From 8eee193940d7371abe549ffe5e48543ac3f05f12 Mon Sep 17 00:00:00 2001 From: Lauros Pajunen Date: Fri, 24 Feb 2023 10:25:58 +0200 Subject: [PATCH 1/6] Implementation of contribution 28 - adaptive transport audio signals for head tracked param-bin --- lib_com/options.h | 4 +- lib_dec/ivas_stat_dec.h | 4 +- lib_rend/ivas_dirac_dec_binaural_functions.c | 95 ++++++++++++++++++++ lib_rend/ivas_hrtf.c | 7 ++ lib_rend/ivas_stat_rend.h | 5 ++ 5 files changed, 112 insertions(+), 3 deletions(-) diff --git a/lib_com/options.h b/lib_com/options.h index 9a6a9d4c5b..2b0f55cf35 100755 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -143,7 +143,7 @@ /*#define FIX_I4_OL_PITCH*/ /* fix open-loop pitch used for EVS core switching */ #define LOW_RATE_TRANS_CORE_CODER /* Eri: Activate low-rate-encoding-of-transients contribution for core coder, affects MC, MASA and SBA */ -#define FIX_197_CREND_INTERFACE +#define FIX_197_CREND_INTERFACE #define FIX_329_ENABLE_TD_RENDERER_REVERB_MC /* Eri: Enable reverb for TD renderer for 5.1 and 7.1 with headtracking enabled for IVAS_dec */ #define FIX_347_DTX_CRASH /* FhG: Fix crash that can happen with DTX */ #define DISABLE_RES_CHANNELS_MCT /* decode only W and residual for Y when outputting to stereo */ @@ -158,6 +158,8 @@ #define SMOOTH_WITH_TRANS_DET #endif +#define NOKIA_ADAPTIVE_BINAURAL_PROTOS + /* ################## End DEVELOPMENT switches ######################### */ /* clang-format on */ #endif diff --git a/lib_dec/ivas_stat_dec.h b/lib_dec/ivas_stat_dec.h index 7c6ac19f7e..06b5775093 100644 --- a/lib_dec/ivas_stat_dec.h +++ b/lib_dec/ivas_stat_dec.h @@ -1217,10 +1217,10 @@ typedef struct Decoder_Struct MONO_DOWNMIX_RENDERER_HANDLE hMonoDmxRenderer; /* Mono downmix structure */ #ifdef FIX_197_CREND_INTERFACE CREND_WRAPPER_HANDLE hCrendWrapper; -#else +#else CREND_HANDLE hCrend; /* Convolution mixer renderer structure */ HRTFS_HANDLE hHrtf; /* HRTFs handle */ -#endif +#endif HRTFS_CREND_HANDLE hSetOfHRTF; /* Set of HRTFs handle (CRend) */ HRTFS_FASTCONV_HANDLE hHrtfFastConv; /* FASTCONV HRTF tables for binaural rendering */ HRTFS_PARAMBIN_HANDLE hHrtfParambin; /* HRTF tables for parametric binauralizer */ diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index 020dd27cfa..a0a0f75efb 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -54,6 +54,14 @@ #define IVAS_TDET_DUCK_MULT_FAC_PARA_BIN ( 2.0f ) #define IVAS_TDET_DUCK_MULT_FAC_PARA_BIN_LOW_BR ( 3.0f ) +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS +#define ADAPT_HTPROTO_IIR_FAC 0.95f +#define ADAPT_HTPROTO_ILD_LIM_DB0 1.0f +#define ADAPT_HTPROTO_ILD_LIM_DB1 4.0f +#define ADAPT_HTPROTO_ROT_LIM_0 0.4f +#define ADAPT_HTPROTO_ROT_LIM_1 0.8f +#endif + /*------------------------------------------------------------------------- * Local function prototypes *------------------------------------------------------------------------*/ @@ -68,6 +76,10 @@ static void ivas_dirac_dec_binaural_determine_processing_matrices( Decoder_Struc static void ivas_dirac_dec_binaural_process_output( Decoder_Struct *st_ivas, float output_f[][L_FRAME48k], float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const int16_t max_band_decorr, const uint8_t numInputChannels, const uint8_t firstSlot, const uint8_t slotEnd ); +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS +static void adaptTransportSignalsHeadtracked( HEAD_TRACK_DATA_HANDLE hHeadTrackData, float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const uint8_t firstSlot, const uint8_t slotEnd, const uint8_t nBins, float Rmat[3][3] ); +#endif + static void ivas_dirac_dec_binaural_check_and_switch_transports_headtracked( HEAD_TRACK_DATA_HANDLE hHeadTrackData, float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const uint8_t firstSlot, const uint8_t slotEnd, const uint8_t nBins, float Rmat[3][3] ); static void formulate2x2MixingMatrix( float Ein1, float Ein2, float CinRe, float CinIm, float Eout1, float Eout2, float CoutRe, float CoutIm, float Q[BINAURAL_CHANNELS][BINAURAL_CHANNELS], float Mre[BINAURAL_CHANNELS][BINAURAL_CHANNELS], float Mim[BINAURAL_CHANNELS][BINAURAL_CHANNELS], const float regularizationFactor ); @@ -562,6 +574,10 @@ static void ivas_dirac_dec_binaural_internal( if ( nchan_transport == 2 ) { +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS + adaptTransportSignalsHeadtracked( st_ivas->hHeadTrackData, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, firstSlot, slotEnd, nBins, Rmat ); +#endif + ivas_dirac_dec_binaural_check_and_switch_transports_headtracked( st_ivas->hHeadTrackData, Cldfb_ImagBuffer_in, Cldfb_RealBuffer_in, firstSlot, slotEnd, nBins, Rmat ); } } @@ -1332,6 +1348,85 @@ static void ivas_dirac_dec_binaural_process_output( } +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS +static void adaptTransportSignalsHeadtracked( + HEAD_TRACK_DATA_HANDLE hHeadTrackData, + float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], + float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], + const uint8_t firstSlot, + const uint8_t slotEnd, + const uint8_t nBins, + float Rmat[3][3] ) +{ + int16_t slot, ch, bin, louderCh; + float re[2], im[2], ILD, mono_factor_ILD, mono_factor_rotation, mono_factor, y_val; + float proc_re[2], proc_im[2], sum_re, sum_im, ene_proc, ene_target, mf; + + /* Determine head-orientation-based mono factor. + Rmat[1][1] entry informs how close the ears are aligned according to transport signals. */ + y_val = 1.0f - fabsf( Rmat[1][1] ); + mono_factor_rotation = ( y_val - ADAPT_HTPROTO_ROT_LIM_0 ) / ( ADAPT_HTPROTO_ROT_LIM_1 - ADAPT_HTPROTO_ROT_LIM_0 ); + mono_factor_rotation = fmaxf( 0.0f, fminf( 1.0f, mono_factor_rotation ) ); + + /* Adapt transport signals in frequency bands */ + for ( slot = firstSlot; slot < slotEnd; slot++ ) + { + float eqVal[60]; + for ( bin = 0; bin < nBins; bin++ ) + { + /* Determine channel energies */ + for ( ch = 0; ch < 2; ch++ ) + { + re[ch] = inRe[ch][slot][bin]; + im[ch] = inIm[ch][slot][bin]; + + hHeadTrackData->chEneIIR[ch][bin] *= ADAPT_HTPROTO_IIR_FAC; + hHeadTrackData->chEneIIR[ch][bin] += ( 1.0f - ADAPT_HTPROTO_IIR_FAC ) * ( ( re[ch] * re[ch] ) + ( im[ch] * im[ch] ) ); + } + + /* Determine ILD */ + ILD = fabsf( 10.0f * log10f( fmaxf( 1e-12f, hHeadTrackData->chEneIIR[0][bin] ) / fmaxf( 1e-12f, hHeadTrackData->chEneIIR[1][bin] ) ) ); + louderCh = ( hHeadTrackData->chEneIIR[1][bin] > hHeadTrackData->chEneIIR[0][bin] ); + + /* Determine ILD-based mono factor */ + mono_factor_ILD = ( ILD - ADAPT_HTPROTO_ILD_LIM_DB0 ) / ( ADAPT_HTPROTO_ILD_LIM_DB1 - ADAPT_HTPROTO_ILD_LIM_DB0 ); + mono_factor_ILD = fmaxf( 0.0f, fminf( 1.0f, mono_factor_ILD ) ); + + /* Combine mono factors */ + mono_factor = mono_factor_ILD * mono_factor_rotation; + + /* Mix original audio and sum signal according to determined mono factor */ + sum_re = re[0] + re[1]; + sum_im = im[0] + im[1]; + for ( ch = 0; ch < 2; ch++ ) + { + mf = ( ch == louderCh ) ? 0.0f : mono_factor; + + proc_re[ch] = mf * sum_re + ( 1.0f - mf ) * re[ch]; + proc_im[ch] = mf * sum_im + ( 1.0f - mf ) * im[ch]; + + hHeadTrackData->procChEneIIR[ch][bin] *= ADAPT_HTPROTO_IIR_FAC; + hHeadTrackData->procChEneIIR[ch][bin] += ( 1.0f - ADAPT_HTPROTO_IIR_FAC ) * ( ( proc_re[ch] * proc_re[ch] ) + ( proc_im[ch] * proc_im[ch] ) ); + } + + /* Equalize */ + ene_target = hHeadTrackData->chEneIIR[0][bin] + hHeadTrackData->chEneIIR[1][bin]; + ene_proc = hHeadTrackData->procChEneIIR[0][bin] + hHeadTrackData->procChEneIIR[1][bin]; + eqVal[bin] = fminf( 4.0f, sqrtf( ene_target / fmaxf( 1e-12f, ene_proc ) ) ); + + for ( ch = 0; ch < 2; ch++ ) + { + inRe[ch][slot][bin] = proc_re[ch] * eqVal[bin]; + inIm[ch][slot][bin] = proc_im[ch] * eqVal[bin]; + } + } + } + + return; +} +#endif + + static void ivas_dirac_dec_binaural_check_and_switch_transports_headtracked( HEAD_TRACK_DATA_HANDLE hHeadTrackData, float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], diff --git a/lib_rend/ivas_hrtf.c b/lib_rend/ivas_hrtf.c index b53c636932..5a38af5514 100644 --- a/lib_rend/ivas_hrtf.c +++ b/lib_rend/ivas_hrtf.c @@ -235,5 +235,12 @@ ivas_error ivas_headTrack_open( ( *hHeadTrackData )->Rmat_prev[i][i] = 1.0f; } +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS + set_zero( ( *hHeadTrackData )->chEneIIR[0], CLDFB_NO_CHANNELS_MAX ); + set_zero( ( *hHeadTrackData )->chEneIIR[1], CLDFB_NO_CHANNELS_MAX ); + set_zero( ( *hHeadTrackData )->procChEneIIR[0], CLDFB_NO_CHANNELS_MAX ); + set_zero( ( *hHeadTrackData )->procChEneIIR[1], CLDFB_NO_CHANNELS_MAX ); +#endif + return IVAS_ERR_OK; } diff --git a/lib_rend/ivas_stat_rend.h b/lib_rend/ivas_stat_rend.h index 586a43c4f8..a00b3db4f0 100644 --- a/lib_rend/ivas_stat_rend.h +++ b/lib_rend/ivas_stat_rend.h @@ -245,6 +245,11 @@ typedef struct ivas_binaural_head_track_struct uint8_t lrSwitchedCurrent; float lrSwitchInterpVal; +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS + float chEneIIR[2][CLDFB_NO_CHANNELS_MAX]; + float procChEneIIR[2][CLDFB_NO_CHANNELS_MAX]; +#endif + int16_t shd_rot_max_order; } HEAD_TRACK_DATA, *HEAD_TRACK_DATA_HANDLE; -- GitLab From b991f4de2917a985b2427488859e0739d084e39d Mon Sep 17 00:00:00 2001 From: Lauros Pajunen Date: Wed, 15 Mar 2023 17:29:30 +0200 Subject: [PATCH 2/6] Computational optimization by reducing tf-resolution --- lib_com/options.h | 1 + lib_rend/ivas_dirac_dec_binaural_functions.c | 129 ++++++++++++++++++- lib_rend/ivas_stat_rend.h | 5 + 3 files changed, 133 insertions(+), 2 deletions(-) diff --git a/lib_com/options.h b/lib_com/options.h index 2b0f55cf35..254774af6e 100755 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -159,6 +159,7 @@ #endif #define NOKIA_ADAPTIVE_BINAURAL_PROTOS +#define NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT /* enable adaptive binaural prototype complexity optimizations */ /* ################## End DEVELOPMENT switches ######################### */ /* clang-format on */ diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index a0a0f75efb..073b56dedd 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -40,6 +40,8 @@ #include "ivas_cnst.h" #include "ivas_rom_binauralRenderer.h" #include "ivas_rom_rend.h" +#include "ivas_rom_com.h" + #ifdef DEBUGGING #include "debug.h" #endif @@ -55,11 +57,17 @@ #define IVAS_TDET_DUCK_MULT_FAC_PARA_BIN_LOW_BR ( 3.0f ) #ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT +/* powf(0.95f, 4.0f) for sub-frame smoothing instead of CLDFB slot */ +#define ADAPT_HTPROTO_IIR_FAC 0.81450625f +#else #define ADAPT_HTPROTO_IIR_FAC 0.95f +#endif + #define ADAPT_HTPROTO_ILD_LIM_DB0 1.0f #define ADAPT_HTPROTO_ILD_LIM_DB1 4.0f -#define ADAPT_HTPROTO_ROT_LIM_0 0.4f -#define ADAPT_HTPROTO_ROT_LIM_1 0.8f +#define ADAPT_HTPROTO_ROT_LIM_0 0.4f +#define ADAPT_HTPROTO_ROT_LIM_1 0.8f #endif /*------------------------------------------------------------------------- @@ -77,8 +85,12 @@ static void ivas_dirac_dec_binaural_determine_processing_matrices( Decoder_Struc static void ivas_dirac_dec_binaural_process_output( Decoder_Struct *st_ivas, float output_f[][L_FRAME48k], float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const int16_t max_band_decorr, const uint8_t numInputChannels, const uint8_t firstSlot, const uint8_t slotEnd ); #ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT +static void adaptTransportSignalsHeadtracked( HEAD_TRACK_DATA_HANDLE hHeadTrackData, float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const uint8_t firstSlot, const uint8_t slotEnd, float Rmat[3][3] ); +#else static void adaptTransportSignalsHeadtracked( HEAD_TRACK_DATA_HANDLE hHeadTrackData, float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const uint8_t firstSlot, const uint8_t slotEnd, const uint8_t nBins, float Rmat[3][3] ); #endif +#endif static void ivas_dirac_dec_binaural_check_and_switch_transports_headtracked( HEAD_TRACK_DATA_HANDLE hHeadTrackData, float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const uint8_t firstSlot, const uint8_t slotEnd, const uint8_t nBins, float Rmat[3][3] ); @@ -575,7 +587,11 @@ static void ivas_dirac_dec_binaural_internal( if ( nchan_transport == 2 ) { #ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT + adaptTransportSignalsHeadtracked( st_ivas->hHeadTrackData, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, firstSlot, slotEnd, Rmat ); +#else adaptTransportSignalsHeadtracked( st_ivas->hHeadTrackData, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, firstSlot, slotEnd, nBins, Rmat ); +#endif #endif ivas_dirac_dec_binaural_check_and_switch_transports_headtracked( st_ivas->hHeadTrackData, Cldfb_ImagBuffer_in, Cldfb_RealBuffer_in, firstSlot, slotEnd, nBins, Rmat ); @@ -1355,12 +1371,19 @@ static void adaptTransportSignalsHeadtracked( float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const uint8_t firstSlot, const uint8_t slotEnd, +#ifndef NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT const uint8_t nBins, +#endif float Rmat[3][3] ) { int16_t slot, ch, bin, louderCh; +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT + float ILD, mono_factor_ILD, mono_factor_rotation, mono_factor, y_val, ene_proc, ene_target; + uint8_t n_slots_per_sf, sf_idx, n_sf; +#else float re[2], im[2], ILD, mono_factor_ILD, mono_factor_rotation, mono_factor, y_val; float proc_re[2], proc_im[2], sum_re, sum_im, ene_proc, ene_target, mf; +#endif /* Determine head-orientation-based mono factor. Rmat[1][1] entry informs how close the ears are aligned according to transport signals. */ @@ -1369,9 +1392,110 @@ static void adaptTransportSignalsHeadtracked( mono_factor_rotation = fmaxf( 0.0f, fminf( 1.0f, mono_factor_rotation ) ); /* Adapt transport signals in frequency bands */ +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT + /* optimization grouping CLDFB bins into MASA bands (they are readily available in ROM and suitable for the task) AND group CLDFB slots into sub-frames */ + n_slots_per_sf = CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES; + n_sf = ( slotEnd - firstSlot ) / n_slots_per_sf; + + for ( sf_idx = 0; sf_idx < n_sf; sf_idx++ ) + { + float eqVal; + uint8_t start_slot, stop_slot; + int16_t band_idx, bin_lo, bin_hi; + + start_slot = firstSlot + sf_idx * n_slots_per_sf; + stop_slot = start_slot + n_slots_per_sf; + + for ( band_idx = 0; band_idx < MASA_FREQUENCY_BANDS; band_idx++ ) + { + float ch_nrg[2]; /* storage for input signal channel energies */ + bin_lo = MASA_band_grouping_24[band_idx]; + bin_hi = MASA_band_grouping_24[band_idx + 1]; + + for ( ch = 0; ch < 2; ch++ ) + { + ch_nrg[ch] = 0.0f; + for ( slot = start_slot; slot < stop_slot; slot++ ) + { + for ( bin = bin_lo; bin < bin_hi; bin++ ) + { + ch_nrg[ch] += ( inRe[ch][slot][bin] * inRe[ch][slot][bin] ) + ( inIm[ch][slot][bin] * inIm[ch][slot][bin] ); + } + } + hHeadTrackData->chEneIIR[ch][band_idx] *= ADAPT_HTPROTO_IIR_FAC; + hHeadTrackData->chEneIIR[ch][band_idx] += ( 1.0f - ADAPT_HTPROTO_IIR_FAC ) * ch_nrg[ch]; + } + + /* Determine ILD */ + ILD = fabsf( 10.0f * log10f( fmaxf( 1e-12f, hHeadTrackData->chEneIIR[0][band_idx] ) / fmaxf( 1e-12f, hHeadTrackData->chEneIIR[1][band_idx] ) ) ); + if ( hHeadTrackData->chEneIIR[1][band_idx] > hHeadTrackData->chEneIIR[0][band_idx] ) + { + louderCh = 1; + } + else + { + louderCh = 0; + } + + /* Determine ILD-based mono factor */ + mono_factor_ILD = ( ILD - ADAPT_HTPROTO_ILD_LIM_DB0 ) / ( ADAPT_HTPROTO_ILD_LIM_DB1 - ADAPT_HTPROTO_ILD_LIM_DB0 ); + mono_factor_ILD = fmaxf( 0.0f, fminf( 1.0f, mono_factor_ILD ) ); + + /* Combine mono factors */ + mono_factor = mono_factor_ILD * mono_factor_rotation; + + /* Mix original audio and sum signal according to determined mono factor */ + for ( ch = 0; ch < 2; ch++ ) + { + if ( ch != louderCh ) + { + float band_nrg = 0.0f; + + for ( slot = start_slot; slot < stop_slot; slot++ ) + { + for ( bin = bin_lo; bin < bin_hi; bin++ ) + { + /* mono sum signal with the computed weight + rest from the original channel */ + inRe[ch][slot][bin] = mono_factor * ( inRe[0][slot][bin] + inRe[1][slot][bin] ) + ( 1.0f - mono_factor ) * inRe[ch][slot][bin]; + inIm[ch][slot][bin] = mono_factor * ( inIm[0][slot][bin] + inIm[1][slot][bin] ) + ( 1.0f - mono_factor ) * inIm[ch][slot][bin]; + band_nrg += ( inRe[ch][slot][bin] * inRe[ch][slot][bin] ) + ( inIm[ch][slot][bin] * inIm[ch][slot][bin] ); + } + } + hHeadTrackData->procChEneIIR[ch][band_idx] *= ADAPT_HTPROTO_IIR_FAC; + hHeadTrackData->procChEneIIR[ch][band_idx] += ( 1.0f - ADAPT_HTPROTO_IIR_FAC ) * band_nrg; + } + else + { + /* processed signal is input. use the original channel, so no need to compute new signals or signal energy */ + hHeadTrackData->procChEneIIR[ch][band_idx] *= ADAPT_HTPROTO_IIR_FAC; + hHeadTrackData->procChEneIIR[ch][band_idx] += ( 1.0f - ADAPT_HTPROTO_IIR_FAC ) * ch_nrg[ch]; + } + } + + /* Equalize */ + ene_target = hHeadTrackData->chEneIIR[0][band_idx] + hHeadTrackData->chEneIIR[1][band_idx]; + ene_proc = hHeadTrackData->procChEneIIR[0][band_idx] + hHeadTrackData->procChEneIIR[1][band_idx]; + eqVal = fminf( 4.0f, sqrtf( ene_target / fmaxf( 1e-12f, ene_proc ) ) ); + + for ( slot = start_slot; slot < stop_slot; slot++ ) + { + for ( ch = 0; ch < 2; ch++ ) + { + for ( bin = bin_lo; bin < bin_hi; bin++ ) + { + inRe[ch][slot][bin] *= eqVal; + inIm[ch][slot][bin] *= eqVal; + } + } + } + } + } +#else + /* original contribution */ for ( slot = firstSlot; slot < slotEnd; slot++ ) { float eqVal[60]; + for ( bin = 0; bin < nBins; bin++ ) { /* Determine channel energies */ @@ -1421,6 +1545,7 @@ static void adaptTransportSignalsHeadtracked( } } } +#endif return; } diff --git a/lib_rend/ivas_stat_rend.h b/lib_rend/ivas_stat_rend.h index a00b3db4f0..51d6ef37ef 100644 --- a/lib_rend/ivas_stat_rend.h +++ b/lib_rend/ivas_stat_rend.h @@ -246,8 +246,13 @@ typedef struct ivas_binaural_head_track_struct float lrSwitchInterpVal; #ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT + float chEneIIR[2][MASA_FREQUENCY_BANDS]; /* independent of the format. MASA bands are suitable for the task and readily available in ROM. */ + float procChEneIIR[2][MASA_FREQUENCY_BANDS]; +#else float chEneIIR[2][CLDFB_NO_CHANNELS_MAX]; float procChEneIIR[2][CLDFB_NO_CHANNELS_MAX]; +#endif #endif int16_t shd_rot_max_order; -- GitLab From 5a699affbd8e6b3c55685d24d6fd8a7aab219a72 Mon Sep 17 00:00:00 2001 From: Lauros Pajunen Date: Thu, 16 Mar 2023 10:29:37 +0200 Subject: [PATCH 3/6] Fix lower sampling rates --- lib_rend/ivas_dirac_dec_binaural_functions.c | 24 ++++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index 073b56dedd..013fb026c9 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -84,13 +84,7 @@ static void ivas_dirac_dec_binaural_determine_processing_matrices( Decoder_Struc static void ivas_dirac_dec_binaural_process_output( Decoder_Struct *st_ivas, float output_f[][L_FRAME48k], float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const int16_t max_band_decorr, const uint8_t numInputChannels, const uint8_t firstSlot, const uint8_t slotEnd ); -#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS -#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT -static void adaptTransportSignalsHeadtracked( HEAD_TRACK_DATA_HANDLE hHeadTrackData, float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const uint8_t firstSlot, const uint8_t slotEnd, float Rmat[3][3] ); -#else static void adaptTransportSignalsHeadtracked( HEAD_TRACK_DATA_HANDLE hHeadTrackData, float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const uint8_t firstSlot, const uint8_t slotEnd, const uint8_t nBins, float Rmat[3][3] ); -#endif -#endif static void ivas_dirac_dec_binaural_check_and_switch_transports_headtracked( HEAD_TRACK_DATA_HANDLE hHeadTrackData, float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const uint8_t firstSlot, const uint8_t slotEnd, const uint8_t nBins, float Rmat[3][3] ); @@ -586,13 +580,7 @@ static void ivas_dirac_dec_binaural_internal( if ( nchan_transport == 2 ) { -#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS -#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT - adaptTransportSignalsHeadtracked( st_ivas->hHeadTrackData, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, firstSlot, slotEnd, Rmat ); -#else adaptTransportSignalsHeadtracked( st_ivas->hHeadTrackData, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, firstSlot, slotEnd, nBins, Rmat ); -#endif -#endif ivas_dirac_dec_binaural_check_and_switch_transports_headtracked( st_ivas->hHeadTrackData, Cldfb_ImagBuffer_in, Cldfb_RealBuffer_in, firstSlot, slotEnd, nBins, Rmat ); } @@ -1380,6 +1368,7 @@ static void adaptTransportSignalsHeadtracked( #ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT float ILD, mono_factor_ILD, mono_factor_rotation, mono_factor, y_val, ene_proc, ene_target; uint8_t n_slots_per_sf, sf_idx, n_sf; + int16_t max_band; #else float re[2], im[2], ILD, mono_factor_ILD, mono_factor_rotation, mono_factor, y_val; float proc_re[2], proc_im[2], sum_re, sum_im, ene_proc, ene_target, mf; @@ -1397,6 +1386,12 @@ static void adaptTransportSignalsHeadtracked( n_slots_per_sf = CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES; n_sf = ( slotEnd - firstSlot ) / n_slots_per_sf; + max_band = 0; + while ( max_band < MASA_FREQUENCY_BANDS && MASA_band_grouping_24[max_band] < nBins ) + { + max_band++; + } + for ( sf_idx = 0; sf_idx < n_sf; sf_idx++ ) { float eqVal; @@ -1406,12 +1401,11 @@ static void adaptTransportSignalsHeadtracked( start_slot = firstSlot + sf_idx * n_slots_per_sf; stop_slot = start_slot + n_slots_per_sf; - for ( band_idx = 0; band_idx < MASA_FREQUENCY_BANDS; band_idx++ ) + for ( band_idx = 0; band_idx < max_band; band_idx++ ) { float ch_nrg[2]; /* storage for input signal channel energies */ bin_lo = MASA_band_grouping_24[band_idx]; - bin_hi = MASA_band_grouping_24[band_idx + 1]; - + bin_hi = min( MASA_band_grouping_24[band_idx + 1], (int16_t) nBins ); for ( ch = 0; ch < 2; ch++ ) { ch_nrg[ch] = 0.0f; -- GitLab From e3e2df4792d688e30bba373ac8a2ba7f30119244 Mon Sep 17 00:00:00 2001 From: Lauros Pajunen Date: Thu, 16 Mar 2023 13:12:19 +0200 Subject: [PATCH 4/6] Enable nBins without flag --- lib_rend/ivas_dirac_dec_binaural_functions.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index 013fb026c9..0fa759bed4 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -1359,9 +1359,7 @@ static void adaptTransportSignalsHeadtracked( float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const uint8_t firstSlot, const uint8_t slotEnd, -#ifndef NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT const uint8_t nBins, -#endif float Rmat[3][3] ) { int16_t slot, ch, bin, louderCh; -- GitLab From 5353d292e59657b8268c79cd45cc18c5d9f3c4df Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Tue, 4 Apr 2023 11:18:13 +0200 Subject: [PATCH 5/6] wrap contribution additions in switch --- lib_rend/ivas_dirac_dec_binaural_functions.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index 0fa759bed4..a0691caca8 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -40,7 +40,9 @@ #include "ivas_cnst.h" #include "ivas_rom_binauralRenderer.h" #include "ivas_rom_rend.h" +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS #include "ivas_rom_com.h" +#endif #ifdef DEBUGGING #include "debug.h" @@ -84,7 +86,9 @@ static void ivas_dirac_dec_binaural_determine_processing_matrices( Decoder_Struc static void ivas_dirac_dec_binaural_process_output( Decoder_Struct *st_ivas, float output_f[][L_FRAME48k], float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const int16_t max_band_decorr, const uint8_t numInputChannels, const uint8_t firstSlot, const uint8_t slotEnd ); +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS static void adaptTransportSignalsHeadtracked( HEAD_TRACK_DATA_HANDLE hHeadTrackData, float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const uint8_t firstSlot, const uint8_t slotEnd, const uint8_t nBins, float Rmat[3][3] ); +#endif static void ivas_dirac_dec_binaural_check_and_switch_transports_headtracked( HEAD_TRACK_DATA_HANDLE hHeadTrackData, float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const uint8_t firstSlot, const uint8_t slotEnd, const uint8_t nBins, float Rmat[3][3] ); @@ -580,7 +584,9 @@ static void ivas_dirac_dec_binaural_internal( if ( nchan_transport == 2 ) { +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS adaptTransportSignalsHeadtracked( st_ivas->hHeadTrackData, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, firstSlot, slotEnd, nBins, Rmat ); +#endif ivas_dirac_dec_binaural_check_and_switch_transports_headtracked( st_ivas->hHeadTrackData, Cldfb_ImagBuffer_in, Cldfb_RealBuffer_in, firstSlot, slotEnd, nBins, Rmat ); } -- GitLab From b769df669a980a8120cb4bd222f89fd0ba55e80e Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Tue, 4 Apr 2023 14:11:04 +0200 Subject: [PATCH 6/6] [fix] crash with headtracking - out of bounds write (array size was changed to MASA_FREQUENCY_BANDS instead of CLDFB_NO_CHANNELS_MAX under _OPT switch) --- lib_rend/ivas_hrtf.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib_rend/ivas_hrtf.c b/lib_rend/ivas_hrtf.c index 5a38af5514..3f5fc08f98 100644 --- a/lib_rend/ivas_hrtf.c +++ b/lib_rend/ivas_hrtf.c @@ -236,10 +236,17 @@ ivas_error ivas_headTrack_open( } #ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT + set_zero( ( *hHeadTrackData )->chEneIIR[0], MASA_FREQUENCY_BANDS ); + set_zero( ( *hHeadTrackData )->chEneIIR[1], MASA_FREQUENCY_BANDS ); + set_zero( ( *hHeadTrackData )->procChEneIIR[0], MASA_FREQUENCY_BANDS ); + set_zero( ( *hHeadTrackData )->procChEneIIR[1], MASA_FREQUENCY_BANDS ); +#else set_zero( ( *hHeadTrackData )->chEneIIR[0], CLDFB_NO_CHANNELS_MAX ); set_zero( ( *hHeadTrackData )->chEneIIR[1], CLDFB_NO_CHANNELS_MAX ); set_zero( ( *hHeadTrackData )->procChEneIIR[0], CLDFB_NO_CHANNELS_MAX ); set_zero( ( *hHeadTrackData )->procChEneIIR[1], CLDFB_NO_CHANNELS_MAX ); +#endif #endif return IVAS_ERR_OK; -- GitLab