diff --git a/lib_com/options.h b/lib_com/options.h index 1ef054819594d418c3deae10230ce406356608fc..1755b11dc8fcbcda71f2dbc4a5b976a182185716 100644 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -154,6 +154,8 @@ #define SBA2MONO /* FhG: Issue 365: Adapt processing of SBA mono output to be in line with stereo output (less delay, lower complexity) */ #define NOKIA_PARAMBIN_REQULARIZATION /* Nokia: Contribution - Configured reqularization factor for parametric binauralizer. */ +#define NOKIA_ADAPTIVE_BINAURAL_PROTOS /* Nokia: Contribution 28: Adaptive binaural prototypes */ +#define NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT /* Nokia: enable adaptive binaural prototype complexity optimizations */ /* ################## End DEVELOPMENT switches ######################### */ /* clang-format on */ diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index b27462d649c10dd186579b1df38858e971beba27..09704b969e21ba1cd443e6ed929fffcf6b459b9f 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -40,6 +40,10 @@ #include "ivas_cnst.h" #include "ivas_rom_binauralRenderer.h" #include "ivas_rom_rend.h" +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS +#include "ivas_rom_com.h" +#endif + #ifdef DEBUGGING #include "debug.h" #endif @@ -54,6 +58,20 @@ #define IVAS_TDET_DUCK_MULT_FAC_PARA_BIN ( 2.0f ) #define IVAS_TDET_DUCK_MULT_FAC_PARA_BIN_LOW_BR ( 3.0f ) +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT +/* powf(0.95f, 4.0f) for sub-frame smoothing instead of CLDFB slot */ +#define ADAPT_HTPROTO_IIR_FAC 0.81450625f +#else +#define ADAPT_HTPROTO_IIR_FAC 0.95f +#endif + +#define ADAPT_HTPROTO_ILD_LIM_DB0 1.0f +#define ADAPT_HTPROTO_ILD_LIM_DB1 4.0f +#define ADAPT_HTPROTO_ROT_LIM_0 0.4f +#define ADAPT_HTPROTO_ROT_LIM_1 0.8f +#endif + /*------------------------------------------------------------------------- * Local function prototypes *------------------------------------------------------------------------*/ @@ -68,6 +86,10 @@ static void ivas_dirac_dec_binaural_determine_processing_matrices( Decoder_Struc static void ivas_dirac_dec_binaural_process_output( Decoder_Struct *st_ivas, float output_f[][L_FRAME48k], float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const int16_t max_band_decorr, const uint8_t numInputChannels, const uint8_t firstSlot, const uint8_t slotEnd ); +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS +static void adaptTransportSignalsHeadtracked( HEAD_TRACK_DATA_HANDLE hHeadTrackData, float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const uint8_t firstSlot, const uint8_t slotEnd, const uint8_t nBins, float Rmat[3][3] ); +#endif + static void ivas_dirac_dec_binaural_check_and_switch_transports_headtracked( HEAD_TRACK_DATA_HANDLE hHeadTrackData, float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const uint8_t firstSlot, const uint8_t slotEnd, const uint8_t nBins, float Rmat[3][3] ); static void formulate2x2MixingMatrix( float Ein1, float Ein2, float CinRe, float CinIm, float Eout1, float Eout2, float CoutRe, float CoutIm, float Q[BINAURAL_CHANNELS][BINAURAL_CHANNELS], float Mre[BINAURAL_CHANNELS][BINAURAL_CHANNELS], float Mim[BINAURAL_CHANNELS][BINAURAL_CHANNELS], const float regularizationFactor ); @@ -554,6 +576,10 @@ static void ivas_dirac_dec_binaural_internal( if ( nchan_transport == 2 ) { +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS + adaptTransportSignalsHeadtracked( st_ivas->hHeadTrackData, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, firstSlot, slotEnd, nBins, Rmat ); +#endif + ivas_dirac_dec_binaural_check_and_switch_transports_headtracked( st_ivas->hHeadTrackData, Cldfb_ImagBuffer_in, Cldfb_RealBuffer_in, firstSlot, slotEnd, nBins, Rmat ); } } @@ -1332,6 +1358,198 @@ static void ivas_dirac_dec_binaural_process_output( } +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS +static void adaptTransportSignalsHeadtracked( + HEAD_TRACK_DATA_HANDLE hHeadTrackData, + float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], + float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], + const uint8_t firstSlot, + const uint8_t slotEnd, + const uint8_t nBins, + float Rmat[3][3] ) +{ + int16_t slot, ch, bin, louderCh; +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT + float ILD, mono_factor_ILD, mono_factor_rotation, mono_factor, y_val, ene_proc, ene_target; + uint8_t n_slots_per_sf, sf_idx, n_sf; + int16_t max_band; +#else + float re[2], im[2], ILD, mono_factor_ILD, mono_factor_rotation, mono_factor, y_val; + float proc_re[2], proc_im[2], sum_re, sum_im, ene_proc, ene_target, mf; +#endif + + /* Determine head-orientation-based mono factor. + Rmat[1][1] entry informs how close the ears are aligned according to transport signals. */ + y_val = 1.0f - fabsf( Rmat[1][1] ); + mono_factor_rotation = ( y_val - ADAPT_HTPROTO_ROT_LIM_0 ) / ( ADAPT_HTPROTO_ROT_LIM_1 - ADAPT_HTPROTO_ROT_LIM_0 ); + mono_factor_rotation = fmaxf( 0.0f, fminf( 1.0f, mono_factor_rotation ) ); + + /* Adapt transport signals in frequency bands */ +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT + /* optimization grouping CLDFB bins into MASA bands (they are readily available in ROM and suitable for the task) AND group CLDFB slots into sub-frames */ + n_slots_per_sf = CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES; + n_sf = ( slotEnd - firstSlot ) / n_slots_per_sf; + + max_band = 0; + while ( max_band < MASA_FREQUENCY_BANDS && MASA_band_grouping_24[max_band] < nBins ) + { + max_band++; + } + + for ( sf_idx = 0; sf_idx < n_sf; sf_idx++ ) + { + float eqVal; + uint8_t start_slot, stop_slot; + int16_t band_idx, bin_lo, bin_hi; + + start_slot = firstSlot + sf_idx * n_slots_per_sf; + stop_slot = start_slot + n_slots_per_sf; + + for ( band_idx = 0; band_idx < max_band; band_idx++ ) + { + float ch_nrg[2]; /* storage for input signal channel energies */ + bin_lo = MASA_band_grouping_24[band_idx]; + bin_hi = min( MASA_band_grouping_24[band_idx + 1], (int16_t) nBins ); + for ( ch = 0; ch < 2; ch++ ) + { + ch_nrg[ch] = 0.0f; + for ( slot = start_slot; slot < stop_slot; slot++ ) + { + for ( bin = bin_lo; bin < bin_hi; bin++ ) + { + ch_nrg[ch] += ( inRe[ch][slot][bin] * inRe[ch][slot][bin] ) + ( inIm[ch][slot][bin] * inIm[ch][slot][bin] ); + } + } + hHeadTrackData->chEneIIR[ch][band_idx] *= ADAPT_HTPROTO_IIR_FAC; + hHeadTrackData->chEneIIR[ch][band_idx] += ( 1.0f - ADAPT_HTPROTO_IIR_FAC ) * ch_nrg[ch]; + } + + /* Determine ILD */ + ILD = fabsf( 10.0f * log10f( fmaxf( 1e-12f, hHeadTrackData->chEneIIR[0][band_idx] ) / fmaxf( 1e-12f, hHeadTrackData->chEneIIR[1][band_idx] ) ) ); + if ( hHeadTrackData->chEneIIR[1][band_idx] > hHeadTrackData->chEneIIR[0][band_idx] ) + { + louderCh = 1; + } + else + { + louderCh = 0; + } + + /* Determine ILD-based mono factor */ + mono_factor_ILD = ( ILD - ADAPT_HTPROTO_ILD_LIM_DB0 ) / ( ADAPT_HTPROTO_ILD_LIM_DB1 - ADAPT_HTPROTO_ILD_LIM_DB0 ); + mono_factor_ILD = fmaxf( 0.0f, fminf( 1.0f, mono_factor_ILD ) ); + + /* Combine mono factors */ + mono_factor = mono_factor_ILD * mono_factor_rotation; + + /* Mix original audio and sum signal according to determined mono factor */ + for ( ch = 0; ch < 2; ch++ ) + { + if ( ch != louderCh ) + { + float band_nrg = 0.0f; + + for ( slot = start_slot; slot < stop_slot; slot++ ) + { + for ( bin = bin_lo; bin < bin_hi; bin++ ) + { + /* mono sum signal with the computed weight + rest from the original channel */ + inRe[ch][slot][bin] = mono_factor * ( inRe[0][slot][bin] + inRe[1][slot][bin] ) + ( 1.0f - mono_factor ) * inRe[ch][slot][bin]; + inIm[ch][slot][bin] = mono_factor * ( inIm[0][slot][bin] + inIm[1][slot][bin] ) + ( 1.0f - mono_factor ) * inIm[ch][slot][bin]; + band_nrg += ( inRe[ch][slot][bin] * inRe[ch][slot][bin] ) + ( inIm[ch][slot][bin] * inIm[ch][slot][bin] ); + } + } + hHeadTrackData->procChEneIIR[ch][band_idx] *= ADAPT_HTPROTO_IIR_FAC; + hHeadTrackData->procChEneIIR[ch][band_idx] += ( 1.0f - ADAPT_HTPROTO_IIR_FAC ) * band_nrg; + } + else + { + /* processed signal is input. use the original channel, so no need to compute new signals or signal energy */ + hHeadTrackData->procChEneIIR[ch][band_idx] *= ADAPT_HTPROTO_IIR_FAC; + hHeadTrackData->procChEneIIR[ch][band_idx] += ( 1.0f - ADAPT_HTPROTO_IIR_FAC ) * ch_nrg[ch]; + } + } + + /* Equalize */ + ene_target = hHeadTrackData->chEneIIR[0][band_idx] + hHeadTrackData->chEneIIR[1][band_idx]; + ene_proc = hHeadTrackData->procChEneIIR[0][band_idx] + hHeadTrackData->procChEneIIR[1][band_idx]; + eqVal = fminf( 4.0f, sqrtf( ene_target / fmaxf( 1e-12f, ene_proc ) ) ); + + for ( slot = start_slot; slot < stop_slot; slot++ ) + { + for ( ch = 0; ch < 2; ch++ ) + { + for ( bin = bin_lo; bin < bin_hi; bin++ ) + { + inRe[ch][slot][bin] *= eqVal; + inIm[ch][slot][bin] *= eqVal; + } + } + } + } + } +#else + /* original contribution */ + for ( slot = firstSlot; slot < slotEnd; slot++ ) + { + float eqVal[60]; + + for ( bin = 0; bin < nBins; bin++ ) + { + /* Determine channel energies */ + for ( ch = 0; ch < 2; ch++ ) + { + re[ch] = inRe[ch][slot][bin]; + im[ch] = inIm[ch][slot][bin]; + + hHeadTrackData->chEneIIR[ch][bin] *= ADAPT_HTPROTO_IIR_FAC; + hHeadTrackData->chEneIIR[ch][bin] += ( 1.0f - ADAPT_HTPROTO_IIR_FAC ) * ( ( re[ch] * re[ch] ) + ( im[ch] * im[ch] ) ); + } + + /* Determine ILD */ + ILD = fabsf( 10.0f * log10f( fmaxf( 1e-12f, hHeadTrackData->chEneIIR[0][bin] ) / fmaxf( 1e-12f, hHeadTrackData->chEneIIR[1][bin] ) ) ); + louderCh = ( hHeadTrackData->chEneIIR[1][bin] > hHeadTrackData->chEneIIR[0][bin] ); + + /* Determine ILD-based mono factor */ + mono_factor_ILD = ( ILD - ADAPT_HTPROTO_ILD_LIM_DB0 ) / ( ADAPT_HTPROTO_ILD_LIM_DB1 - ADAPT_HTPROTO_ILD_LIM_DB0 ); + mono_factor_ILD = fmaxf( 0.0f, fminf( 1.0f, mono_factor_ILD ) ); + + /* Combine mono factors */ + mono_factor = mono_factor_ILD * mono_factor_rotation; + + /* Mix original audio and sum signal according to determined mono factor */ + sum_re = re[0] + re[1]; + sum_im = im[0] + im[1]; + for ( ch = 0; ch < 2; ch++ ) + { + mf = ( ch == louderCh ) ? 0.0f : mono_factor; + + proc_re[ch] = mf * sum_re + ( 1.0f - mf ) * re[ch]; + proc_im[ch] = mf * sum_im + ( 1.0f - mf ) * im[ch]; + + hHeadTrackData->procChEneIIR[ch][bin] *= ADAPT_HTPROTO_IIR_FAC; + hHeadTrackData->procChEneIIR[ch][bin] += ( 1.0f - ADAPT_HTPROTO_IIR_FAC ) * ( ( proc_re[ch] * proc_re[ch] ) + ( proc_im[ch] * proc_im[ch] ) ); + } + + /* Equalize */ + ene_target = hHeadTrackData->chEneIIR[0][bin] + hHeadTrackData->chEneIIR[1][bin]; + ene_proc = hHeadTrackData->procChEneIIR[0][bin] + hHeadTrackData->procChEneIIR[1][bin]; + eqVal[bin] = fminf( 4.0f, sqrtf( ene_target / fmaxf( 1e-12f, ene_proc ) ) ); + + for ( ch = 0; ch < 2; ch++ ) + { + inRe[ch][slot][bin] = proc_re[ch] * eqVal[bin]; + inIm[ch][slot][bin] = proc_im[ch] * eqVal[bin]; + } + } + } +#endif + + return; +} +#endif + + static void ivas_dirac_dec_binaural_check_and_switch_transports_headtracked( HEAD_TRACK_DATA_HANDLE hHeadTrackData, float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], diff --git a/lib_rend/ivas_rotation.c b/lib_rend/ivas_rotation.c index 0386a49c7d9fdaa727c0bee1f0c40563218f4adb..58536df19f343eff092a44f56a2f31d3b15c6739 100644 --- a/lib_rend/ivas_rotation.c +++ b/lib_rend/ivas_rotation.c @@ -85,6 +85,20 @@ ivas_error ivas_headTrack_open( ( *hHeadTrackData )->Rmat_prev[i][i] = 1.0f; } +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT + set_zero( ( *hHeadTrackData )->chEneIIR[0], MASA_FREQUENCY_BANDS ); + set_zero( ( *hHeadTrackData )->chEneIIR[1], MASA_FREQUENCY_BANDS ); + set_zero( ( *hHeadTrackData )->procChEneIIR[0], MASA_FREQUENCY_BANDS ); + set_zero( ( *hHeadTrackData )->procChEneIIR[1], MASA_FREQUENCY_BANDS ); +#else + set_zero( ( *hHeadTrackData )->chEneIIR[0], CLDFB_NO_CHANNELS_MAX ); + set_zero( ( *hHeadTrackData )->chEneIIR[1], CLDFB_NO_CHANNELS_MAX ); + set_zero( ( *hHeadTrackData )->procChEneIIR[0], CLDFB_NO_CHANNELS_MAX ); + set_zero( ( *hHeadTrackData )->procChEneIIR[1], CLDFB_NO_CHANNELS_MAX ); +#endif +#endif + return IVAS_ERR_OK; } diff --git a/lib_rend/ivas_stat_rend.h b/lib_rend/ivas_stat_rend.h index c94b55f3ccb921afee07d304115ca69bc8067927..e7ff9eb0bafafad55ac6ceb4fc5535955937b0dd 100644 --- a/lib_rend/ivas_stat_rend.h +++ b/lib_rend/ivas_stat_rend.h @@ -268,6 +268,16 @@ typedef struct ivas_binaural_head_track_struct uint8_t lrSwitchedCurrent; float lrSwitchInterpVal; +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS +#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT + float chEneIIR[2][MASA_FREQUENCY_BANDS]; /* independent of the format. MASA bands are suitable for the task and readily available in ROM. */ + float procChEneIIR[2][MASA_FREQUENCY_BANDS]; +#else + float chEneIIR[2][CLDFB_NO_CHANNELS_MAX]; + float procChEneIIR[2][CLDFB_NO_CHANNELS_MAX]; +#endif +#endif + int16_t shd_rot_max_order; ivas_orient_trk_state_t *OrientationTracker;