From 60ef3077b92b3832e6be2f92933f69bdd73d2192 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Thu, 23 Feb 2023 15:34:13 +0100 Subject: [PATCH 1/2] Enable reverb for TD renderer for ISM, 5.1 and 7.1 with headtracking enabled for IVAS_rend - under FIX_330_ENABLE_TD_RENDERER_REVERB_REND define - Also fix bug affecting TD renderer reverb introduced in commit c5de9c9a --- lib_com/options.h | 2 + lib_dec/ivas_init_dec.c | 14 + lib_dec/ivas_objectRenderer_internal.c | 9 +- lib_dec/ivas_stat_dec.h | 7 +- lib_rend/ivas_objectRenderer.c | 62 ++++- lib_rend/ivas_prot_rend.h | 9 + lib_rend/ivas_reverb.c | 11 + lib_rend/lib_rend.c | 367 ++++++++++++++++++------- 8 files changed, 370 insertions(+), 111 deletions(-) diff --git a/lib_com/options.h b/lib_com/options.h index 9a6a9d4c5b..18e4f48b55 100755 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -145,6 +145,8 @@ #define LOW_RATE_TRANS_CORE_CODER /* Eri: Activate low-rate-encoding-of-transients contribution for core coder, affects MC, MASA and SBA */ #define FIX_197_CREND_INTERFACE #define FIX_329_ENABLE_TD_RENDERER_REVERB_MC /* Eri: Enable reverb for TD renderer for 5.1 and 7.1 with headtracking enabled for IVAS_dec */ +#define FIX_330_ENABLE_TD_RENDERER_REVERB_REND /* Eri: Enable reverb for TD renderer for ISM, 5.1 and 7.1 with headtracking enabled for IVAS_rend */ + #define FIX_347_DTX_CRASH /* FhG: Fix crash that can happen with DTX */ #define DISABLE_RES_CHANNELS_MCT /* decode only W and residual for Y when outputting to stereo */ #define FIX_107_5MS_SUBFRAME_RENDERING diff --git a/lib_dec/ivas_init_dec.c b/lib_dec/ivas_init_dec.c index 7f28bc4226..c0b5a9805f 100644 --- a/lib_dec/ivas_init_dec.c +++ b/lib_dec/ivas_init_dec.c @@ -1210,6 +1210,12 @@ ivas_error ivas_init_decoder( if ( st_ivas->hRenderConfig->roomAcoustics.late_reverb_on ) { +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + if ( ( error = ivas_reverb_open( &st_ivas->hReverb, st_ivas->hDecoderConfig->output_config, NULL, st_ivas->hRenderConfig, st_ivas->hDecoderConfig->output_Fs ) ) != IVAS_ERR_OK ) + { + return error; + } +#else #ifdef FIX_197_CREND_INTERFACE #ifdef FIX_329_ENABLE_TD_RENDERER_REVERB_MC if ( ( error = ivas_rend_initCrendWrapper( &st_ivas->hCrendWrapper ) ) != IVAS_ERR_OK ) @@ -1226,6 +1232,7 @@ ivas_error ivas_init_decoder( { return IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for Crend\n" ); } +#endif } } else if ( st_ivas->renderer_type == RENDERER_MC ) @@ -1579,6 +1586,9 @@ void ivas_initialize_handles_dec( st_ivas->hMonoDmxRenderer = NULL; #ifdef FIX_197_CREND_INTERFACE st_ivas->hCrendWrapper = NULL; +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + st_ivas->hReverb = NULL; +#endif #else st_ivas->hCrend = NULL; st_ivas->hHrtf = NULL; @@ -1745,6 +1755,10 @@ void ivas_destroy_dec( #else ivas_crend_close( st_ivas ); #endif +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + /* Reverb handle */ + ivas_reverb_close( &st_ivas->hReverb ); +#endif /* LS config converter handle */ ivas_ls_setup_conversion_close( &st_ivas->hLsSetUpConversion ); diff --git a/lib_dec/ivas_objectRenderer_internal.c b/lib_dec/ivas_objectRenderer_internal.c index d2f08ffdd6..09db6fabfb 100644 --- a/lib_dec/ivas_objectRenderer_internal.c +++ b/lib_dec/ivas_objectRenderer_internal.c @@ -71,13 +71,20 @@ void ivas_td_binaural_renderer( const int16_t output_frame /* i : output frame length */ ) { +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + ivas_td_binaural_renderer_unwrap( st_ivas->hReverb, +#else ivas_td_binaural_renderer_unwrap( st_ivas->hRenderConfig, st_ivas->ini_frame, #ifdef FIX_197_CREND_INTERFACE st_ivas->hCrendWrapper, #else st_ivas->hCrend, +#endif #endif st_ivas->transport_config, - st_ivas->hDecoderConfig->output_Fs, st_ivas->hBinRendererTd, st_ivas->nchan_transport, LFE_CHANNEL, st_ivas->ivas_format, +#ifndef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + st_ivas->hDecoderConfig->output_Fs, +#endif + st_ivas->hBinRendererTd, st_ivas->nchan_transport, LFE_CHANNEL, st_ivas->ivas_format, st_ivas->hIsmMetaData, st_ivas->hDecoderConfig->Opt_Headrotation, ( st_ivas->hHeadTrackData != NULL ) ? st_ivas->hHeadTrackData->Quaternions : NULL, output, output_frame ); } diff --git a/lib_dec/ivas_stat_dec.h b/lib_dec/ivas_stat_dec.h index 7c6ac19f7e..574a96abc9 100644 --- a/lib_dec/ivas_stat_dec.h +++ b/lib_dec/ivas_stat_dec.h @@ -1217,10 +1217,13 @@ typedef struct Decoder_Struct MONO_DOWNMIX_RENDERER_HANDLE hMonoDmxRenderer; /* Mono downmix structure */ #ifdef FIX_197_CREND_INTERFACE CREND_WRAPPER_HANDLE hCrendWrapper; -#else +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + REVERB_HANDLE hReverb; /* Reverb handle */ +#endif +#else CREND_HANDLE hCrend; /* Convolution mixer renderer structure */ HRTFS_HANDLE hHrtf; /* HRTFs handle */ -#endif +#endif HRTFS_CREND_HANDLE hSetOfHRTF; /* Set of HRTFs handle (CRend) */ HRTFS_FASTCONV_HANDLE hHrtfFastConv; /* FASTCONV HRTF tables for binaural rendering */ HRTFS_PARAMBIN_HANDLE hHrtfParambin; /* HRTF tables for parametric binauralizer */ diff --git a/lib_rend/ivas_objectRenderer.c b/lib_rend/ivas_objectRenderer.c index b60967e7f7..69a8c88219 100644 --- a/lib_rend/ivas_objectRenderer.c +++ b/lib_rend/ivas_objectRenderer.c @@ -221,15 +221,21 @@ void ivas_td_binaural_close( *---------------------------------------------------------------------*/ void ivas_td_binaural_renderer_unwrap( - RENDER_CONFIG_DATA *hRenderConfig, /* i : Renderer configuration */ - const int16_t ini_frame, /* i : Initialization frame counter */ +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + REVERB_HANDLE hReverb, /* i : reverb handle */ +#else + RENDER_CONFIG_DATA *hRenderConfig, /* i : Renderer configuration */ + const int16_t ini_frame, /* i : Initialization frame counter */ #ifdef FIX_197_CREND_INTERFACE CREND_WRAPPER_HANDLE hCrendWrapper, /* i : Crend wrapper handle */ #else CREND_HANDLE hCrend, /* i : Crend handle */ #endif - AUDIO_CONFIG transport_config, /* i : Transport configuration */ - const int32_t output_Fs, /* i : Output sampling rate */ +#endif + AUDIO_CONFIG transport_config, /* i : Transport configuration */ +#ifndef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + const int32_t output_Fs, /* i : Output sampling rate */ +#endif BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD binaural object renderer handle */ const int16_t nchan_transport, /* i : Transport channels (ISms) */ const int16_t lfe_idx, /* i : LFE channel index */ @@ -246,6 +252,7 @@ void ivas_td_binaural_renderer_unwrap( float reverb_signal[BINAURAL_CHANNELS][L_FRAME48k]; subframe_length = output_frame / MAX_PARAM_SPATIAL_SUBFRAMES; +#ifndef FIX_330_ENABLE_TD_RENDERER_REVERB_REND if ( hRenderConfig != NULL ) { @@ -263,6 +270,7 @@ void ivas_td_binaural_renderer_unwrap( output_Fs ); } } +#endif /* Update object position(s) */ TDREND_Update_object_positions( hBinRendererTd, nchan_transport, lfe_idx, ivas_format, hIsmMetaData, output ); @@ -272,11 +280,19 @@ void ivas_td_binaural_renderer_unwrap( /* Update the listener's location/orientation */ TDREND_Update_listener_orientation( hBinRendererTd, Opt_Headrotation, ( Quaternions != NULL ) ? &Quaternions[subframe_idx] : NULL ); +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + if ( ( hReverb != NULL ) && ( hReverb->pConfig.roomAcoustics.late_reverb_on ) ) +#else if ( ( hRenderConfig != NULL ) && ( hRenderConfig->roomAcoustics.late_reverb_on ) ) +#endif { ivas_reverb_process( #ifdef FIX_197_CREND_INTERFACE +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + hReverb, +#else hCrendWrapper->hCrend->hReverb, +#endif #else hCrend->hReverb, #endif @@ -286,6 +302,7 @@ void ivas_td_binaural_renderer_unwrap( reverb_signal, subframe_idx ); +#ifndef FIX_330_ENABLE_TD_RENDERER_REVERB_REND ivas_reverb_process( #ifdef FIX_197_CREND_INTERFACE hCrendWrapper->hCrend->hReverb, @@ -297,15 +314,26 @@ void ivas_td_binaural_renderer_unwrap( output, reverb_signal, subframe_idx ); +#endif } /* Render subframe */ TDREND_GetMix( hBinRendererTd, output, subframe_length, subframe_idx ); } + +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + if ( hReverb != NULL ) +#else if ( hRenderConfig != NULL ) /* Renderer Configuration not enabled in TD standalone renderer */ +#endif { + +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + if ( hReverb != NULL ) +#else if ( hRenderConfig->roomAcoustics.late_reverb_on ) +#endif { /* add reverb to rendered signals */ v_add( reverb_signal[0], output[0], output[0], output_frame ); @@ -577,20 +605,27 @@ ivas_error ivas_td_binaural_renderer_ext( const LSSETUP_CUSTOM_STRUCT *customLsInput, /* i : Input custom loudspeaker layout */ const IVAS_REND_HeadRotData *headRotData, /* i : Input head positions */ const IVAS_REND_AudioObjectPosition *currentPos, /* i : Object position */ - const int16_t output_frame, /* i : output frame length */ - float output[][L_FRAME48k] /* i/o: SCE channels / Binaural synthesis */ +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + const REVERB_HANDLE reverb, /* i : reverb handle */ +#endif + const int16_t output_frame, /* i : output frame length */ + float output[][L_FRAME48k] /* i/o: SCE channels / Binaural synthesis */ ) { ISM_METADATA_FRAME hIsmMetaDataFrame; ISM_METADATA_HANDLE hIsmMetaData[1]; int16_t lfe_idx; int16_t num_src; +#ifndef FIX_330_ENABLE_TD_RENDERER_REVERB_REND /* TODO tmu : pass down renderer config struct */ // float reverb_signal[BINAURAL_CHANNELS][L_FRAME48k]; +#endif IVAS_FORMAT ivas_format; IVAS_REND_AudioConfigType inConfigType; AUDIO_CONFIG transport_config; +#ifndef FIX_330_ENABLE_TD_RENDERER_REVERB_REND int32_t output_Fs; +#endif push_wmops( "ivas_td_binaural_renderer_ext" ); @@ -601,6 +636,9 @@ ivas_error ivas_td_binaural_renderer_ext( if ( inConfigType == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) { ivas_format = MC_FORMAT; +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + transport_config = getIvasAudioConfigFromRendAudioConfig( inConfig ); +#endif if ( inConfig != IVAS_REND_AUDIO_CONFIG_LS_CUSTOM ) { getAudioConfigNumChannels( inConfig, &num_src ); @@ -615,22 +653,32 @@ ivas_error ivas_td_binaural_renderer_ext( { ivas_format = ISM_FORMAT; num_src = 1; +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + transport_config = AUDIO_CONFIG_ISM1; +#endif hIsmMetaData[0] = &hIsmMetaDataFrame; hIsmMetaData[0]->azimuth = currentPos->azimuth; hIsmMetaData[0]->elevation = currentPos->elevation; } +#ifndef FIX_330_ENABLE_TD_RENDERER_REVERB_REND #ifdef FIX_197_CREND_INTERFACE transport_config = getIvasAudioConfigFromRendAudioConfig( inConfig ); #else transport_config = rendAudioConfigToIvasAudioConfig( inConfig ); #endif output_Fs = output_frame * 50; - +#endif +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + ivas_td_binaural_renderer_unwrap( reverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, + ivas_format, hIsmMetaData, headRotData->headRotEnabled, ( headRotData != NULL ) ? headRotData->headPositions : NULL, + output, output_frame ); +#else ivas_td_binaural_renderer_unwrap( NULL, 1, NULL, transport_config, output_Fs, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, ( headRotData != NULL ) ? headRotData->headPositions : NULL, output, output_frame ); +#endif pop_wmops(); diff --git a/lib_rend/ivas_prot_rend.h b/lib_rend/ivas_prot_rend.h index 8a6cb1a2f9..a2e14a35f1 100644 --- a/lib_rend/ivas_prot_rend.h +++ b/lib_rend/ivas_prot_rend.h @@ -216,15 +216,21 @@ void ivas_HRTF_CRend_binary_close( *----------------------------------------------------------------------------------*/ void ivas_td_binaural_renderer_unwrap( +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + REVERB_HANDLE hReverb, /* i : reverb handle */ +#else RENDER_CONFIG_DATA *hRenderConfig, /* i : Renderer configuration */ const int16_t ini_frame, /* i : Initialization frame counter */ #ifdef FIX_197_CREND_INTERFACE CREND_WRAPPER_HANDLE hCrendWrapper, /* i : Crend wrapper handle */ #else CREND_HANDLE hCrend, /* i : Crend handle */ +#endif #endif AUDIO_CONFIG transport_config, /* i : Transport configuration */ +#ifndef FIX_330_ENABLE_TD_RENDERER_REVERB_REND const int32_t output_Fs, /* i : Output sampling rate */ +#endif BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD binaural object renderer handle */ const int16_t nchan_transport, /* i : Transport channels (ISms) */ const int16_t lfe_idx, /* i : LFE channel index */ @@ -242,6 +248,9 @@ ivas_error ivas_td_binaural_renderer_ext( const LSSETUP_CUSTOM_STRUCT *customLsInput, /* i : Input custom loudspeaker layout */ const IVAS_REND_HeadRotData *headRotData, /* i : Input head positions */ const IVAS_REND_AudioObjectPosition *currentPos, /* i : Object position */ +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + const REVERB_HANDLE reverb, /* i : reverb handle */ +#endif const int16_t output_frame, /* i : output frame length */ float output[][L_FRAME48k] /* i/o: SCE channels / Binaural synthesis */ ); diff --git a/lib_rend/ivas_reverb.c b/lib_rend/ivas_reverb.c index a3d55584b4..3911c36601 100644 --- a/lib_rend/ivas_reverb.c +++ b/lib_rend/ivas_reverb.c @@ -1175,6 +1175,17 @@ ivas_error ivas_reverb_open( /* set up reverb acoustic data on the basis of HRTF data and renderer config */ set_reverb_acoustic_data( ¶ms, input_audio_config, hHrtf, &hRenderConfig->roomAcoustics, subframe_len, nr_fc_input, nr_fc_fft_filter ); +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + /* set reverb acoustic configuration based on renderer config */ +#ifdef DEBUGGING + pState->pConfig.renderer_type_override = hRenderConfig->renderer_type_override; +#endif + pState->pConfig.roomAcoustics.override = hRenderConfig->roomAcoustics.override; + pState->pConfig.roomAcoustics.use_brir = hRenderConfig->roomAcoustics.use_brir; + pState->pConfig.roomAcoustics.late_reverb_on = hRenderConfig->roomAcoustics.late_reverb_on; + pState->pConfig.roomAcoustics.nBands = hRenderConfig->roomAcoustics.nBands; +#endif + /* set up input downmix */ pState->dmx_gain = calc_dmx_gain(); diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index 8f868bd58f..788c439e62 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -117,6 +117,9 @@ typedef struct TDREND_WRAPPER tdRendWrapper; #ifdef FIX_197_CREND_INTERFACE CREND_WRAPPER_HANDLE crendWrapper; +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + REVERB_HANDLE reverb; +#endif #else CREND_WRAPPER crendWrapper; #endif @@ -146,6 +149,9 @@ typedef struct TDREND_WRAPPER tdRendWrapper; #ifdef FIX_197_CREND_INTERFACE CREND_WRAPPER_HANDLE crendWrapper; +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + REVERB_HANDLE reverb; +#endif #else CREND_WRAPPER crendWrapper; #endif @@ -1062,6 +1068,9 @@ static ivas_error setRendInputActiveIsm( inputIsm->previousPos = defaultObjectPosition(); #ifdef FIX_197_CREND_INTERFACE inputIsm->crendWrapper = NULL; +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + inputIsm->reverb = NULL; +#endif #else inputIsm->crendWrapper = defaultCrendWrapper(); #endif @@ -1075,19 +1084,38 @@ static ivas_error setRendInputActiveIsm( } else if ( outConfig == IVAS_REND_AUDIO_CONFIG_BINAURAL_ROOM ) { - error = ivas_rend_openCrend( &inputIsm->crendWrapper, - AUDIO_CONFIG_7_1_4, +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + if ( hRendCfg != NULL && hRendCfg->roomAcoustics.use_brir == 0 && hRendCfg->roomAcoustics.late_reverb_on ) + { + error = ivas_td_binaural_open_ext( &inputIsm->tdRendWrapper, inConfig, NULL, *rendCtx.pOutSampleRate ); + if ( ( error = ivas_reverb_open( &( inputIsm->reverb ), + getIvasAudioConfigFromRendAudioConfig( outConfig ), + NULL, + hRendCfg, + *rendCtx.pOutSampleRate ) ) != IVAS_ERR_OK ) + { + return error; + } + } + else + { +#endif + error = ivas_rend_openCrend( &inputIsm->crendWrapper, + AUDIO_CONFIG_7_1_4, #ifdef FIX_197_CREND_INTERFACE - getIvasAudioConfigFromRendAudioConfig( outConfig ), + getIvasAudioConfigFromRendAudioConfig( outConfig ), #else rendAudioConfigToIvasAudioConfig( outConfig ), #endif - hRendCfg, + hRendCfg, #ifdef FIX_197_CREND_INTERFACE - 0, + 0, +#endif + NULL, + *rendCtx.pOutSampleRate ); +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + } #endif - NULL, - *rendCtx.pOutSampleRate ); } if ( error != IVAS_ERR_OK ) { @@ -1115,6 +1143,12 @@ static void clearInputIsm( { ivas_rend_closeCrend( &inputIsm->crendWrapper ); } +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + if ( inputIsm->reverb != NULL ) + { + ivas_reverb_close( &inputIsm->reverb ); + } +#endif if ( inputIsm->tdRendWrapper.hBinRendererTd != NULL ) { ivas_td_binaural_close( &inputIsm->tdRendWrapper.hBinRendererTd ); @@ -1754,6 +1788,12 @@ static ivas_error initMcBinauralRendering( { ivas_rend_closeCrend( &inputMc->crendWrapper ); } +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + if ( inputMc->reverb != NULL ) + { + ivas_reverb_close( &inputMc->reverb ); + } +#endif if ( inputMc->efapInWrapper.hEfap != NULL ) { efap_free_data( &inputMc->efapInWrapper.hEfap ); @@ -1783,6 +1823,19 @@ static ivas_error initMcBinauralRendering( { return error; } +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + if ( hRendCfg != NULL && hRendCfg->roomAcoustics.use_brir == 0 && hRendCfg->roomAcoustics.late_reverb_on ) + { + if ( ( error = ivas_reverb_open( &( inputMc->reverb ), + getIvasAudioConfigFromRendAudioConfig( outConfig ), + NULL, + hRendCfg, + outSampleRate ) ) != IVAS_ERR_OK ) + { + return error; + } + } +#endif } { @@ -1893,6 +1946,9 @@ static ivas_error setRendInputActiveMc( inputMc->tdRendWrapper = defaultTdRendWrapper(); #ifdef FIX_197_CREND_INTERFACE inputMc->crendWrapper = NULL; +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + inputMc->reverb = NULL; +#endif #else inputMc->crendWrapper = defaultCrendWrapper(); #endif @@ -1937,6 +1993,12 @@ static void clearInputMc( { ivas_rend_closeCrend( &inputMc->crendWrapper ); } +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + if ( inputMc->reverb != NULL ) + { + ivas_reverb_close( &inputMc->reverb ); + } +#endif if ( inputMc->tdRendWrapper.hBinRendererTd != NULL ) { ivas_td_binaural_close( &inputMc->tdRendWrapper.hBinRendererTd ); @@ -2699,6 +2761,9 @@ ivas_error IVAS_REND_Open( hIvasRend->inputsIsm[i].crendWrapper = NULL; #else hIvasRend->inputsIsm[i].crendWrapper.hCrend = NULL; +#endif +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + hIvasRend->inputsIsm[i].reverb = NULL; #endif hIvasRend->inputsIsm[i].tdRendWrapper.hBinRendererTd = NULL; } @@ -2710,6 +2775,9 @@ ivas_error IVAS_REND_Open( hIvasRend->inputsMc[i].crendWrapper = NULL; #else hIvasRend->inputsMc[i].crendWrapper.hCrend = NULL; +#endif +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + hIvasRend->inputsMc[i].reverb = NULL; #endif hIvasRend->inputsMc[i].tdRendWrapper.hBinRendererTd = NULL; } @@ -4182,6 +4250,9 @@ static ivas_error renderIsmToBinaural( NULL, ismInput->base.ctx.pHeadRotData, &ismInput->currentPos, +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + ismInput->reverb, +#endif outAudio.config.numSamplesPerChannel, tmpTDRendBuffer ) ) != IVAS_ERR_OK ) { @@ -4204,7 +4275,11 @@ static ivas_error renderIsmToBinauralRoom( int16_t subframe_idx, subframe_len; int16_t tmp; rotation_matrix Rmat; +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + float tmpRendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; +#else float tmpCrendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; +#endif IVAS_QUATERNION quat; ivas_error error; pan_vector currentPanGains; @@ -4218,94 +4293,132 @@ static ivas_error renderIsmToBinauralRoom( headRotData = ismInput->base.ctx.pHeadRotData; rotatedPos = defaultObjectPosition(); - if ( headRotData->headRotEnabled ) +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + if ( ismInput->reverb != NULL && ismInput->reverb->pConfig.roomAcoustics.use_brir == 0 && ismInput->reverb->pConfig.roomAcoustics.late_reverb_on == 1 && headRotData->headRotEnabled ) { - subframe_len = ismInput->base.inputBuffer.config.numSamplesPerChannel / RENDERER_HEAD_POSITIONS_PER_FRAME; - // for ( subframe_idx = 0; subframe_idx < RENDERER_HEAD_POSITIONS_PER_FRAME; subframe_idx++ ) - for ( subframe_idx = 0; subframe_idx < 1; subframe_idx++ ) - { - quat.w = headRotData->headPositions[subframe_idx].w; - quat.x = headRotData->headPositions[subframe_idx].x; - quat.y = headRotData->headPositions[subframe_idx].y; - quat.z = headRotData->headPositions[subframe_idx].z; + copyBufferTo2dArray( ismInput->base.inputBuffer, tmpRendBuffer ); - QuatToRotMat( quat, Rmat ); + /* TODO tmu : missing: interpolation between positions, 5ms rendering */ + if ( ( error = ivas_td_binaural_renderer_ext( &ismInput->tdRendWrapper, + ismInput->base.inConfig, + NULL, + ismInput->base.ctx.pHeadRotData, + &ismInput->currentPos, +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + ismInput->reverb, +#endif + outAudio.config.numSamplesPerChannel, + tmpRendBuffer ) ) != IVAS_ERR_OK ) + { + return error; } - (void) subframe_len; // avoid warning - } - - /* TODO tmu : missing: interpolation between positions, 5ms rendering */ - /* TODO(sgi): Possible optimization: less processing needed if position didn't change */ - /* TODO tmu2sgi: needs a lot of cleanup, we could also add rot_gains_prev to ismInput and use that */ - /* previous position gains */ - if ( headRotData->headRotEnabled ) - { - rotateAziEle( ismInput->previousPos.azimuth, ismInput->previousPos.elevation, &azi_rot, &ele_rot, ismInput->rot_mat_prev, 0 ); - rotatedPos.azimuth = (float) azi_rot; - rotatedPos.elevation = (float) ele_rot; } - if ( ( error = getEfapGains( *ismInput->base.ctx.pEfapOutWrapper, - ( headRotData->headRotEnabled ) ? rotatedPos.azimuth : ismInput->previousPos.azimuth, - ( headRotData->headRotEnabled ) ? rotatedPos.elevation : ismInput->previousPos.elevation, - previousPanGains ) ) != IVAS_ERR_OK ) + else { - return error; - } +#endif - /* current position gains */ - if ( headRotData->headRotEnabled ) - { - rotateAziEle( ismInput->currentPos.azimuth, ismInput->currentPos.elevation, &azi_rot, &ele_rot, Rmat, 0 ); - rotatedPos.azimuth = (float) azi_rot; - rotatedPos.elevation = (float) ele_rot; - } - if ( ( error = getEfapGains( *ismInput->base.ctx.pEfapOutWrapper, - ( headRotData->headRotEnabled ) ? rotatedPos.azimuth : ismInput->currentPos.azimuth, - ( headRotData->headRotEnabled ) ? rotatedPos.elevation : ismInput->currentPos.elevation, - currentPanGains ) ) != IVAS_ERR_OK ) - { - return error; - } + if ( headRotData->headRotEnabled ) + { + subframe_len = ismInput->base.inputBuffer.config.numSamplesPerChannel / RENDERER_HEAD_POSITIONS_PER_FRAME; + // for ( subframe_idx = 0; subframe_idx < RENDERER_HEAD_POSITIONS_PER_FRAME; subframe_idx++ ) + for ( subframe_idx = 0; subframe_idx < 1; subframe_idx++ ) + { + quat.w = headRotData->headPositions[subframe_idx].w; + quat.x = headRotData->headPositions[subframe_idx].x; + quat.y = headRotData->headPositions[subframe_idx].y; + quat.z = headRotData->headPositions[subframe_idx].z; - for ( i = 0; i < 3; i++ ) - { - mvr2r( Rmat[i], ismInput->rot_mat_prev[i], 3 ); - } + QuatToRotMat( quat, Rmat ); + } + (void) subframe_len; // avoid warning + } - /* intermediate rendering to 7_1_4 */ - tmpMcBuffer = ismInput->base.inputBuffer; - getAudioConfigNumChannels( IVAS_REND_AUDIO_CONFIG_7_1_4, &tmp ); - tmpMcBuffer.config.numChannels = tmp; - tmpMcBuffer.data = malloc( tmpMcBuffer.config.numSamplesPerChannel * tmpMcBuffer.config.numChannels * sizeof( float ) ); - set_zero( tmpMcBuffer.data, tmpMcBuffer.config.numSamplesPerChannel * tmpMcBuffer.config.numChannels ); + /* TODO tmu : missing: interpolation between positions, 5ms rendering */ + /* TODO(sgi): Possible optimization: less processing needed if position didn't change */ + /* TODO tmu2sgi: needs a lot of cleanup, we could also add rot_gains_prev to ismInput and use that */ + /* previous position gains */ + if ( headRotData->headRotEnabled ) + { + rotateAziEle( ismInput->previousPos.azimuth, ismInput->previousPos.elevation, &azi_rot, &ele_rot, ismInput->rot_mat_prev, 0 ); + rotatedPos.azimuth = (float) azi_rot; + rotatedPos.elevation = (float) ele_rot; + } + if ( ( error = getEfapGains( *ismInput->base.ctx.pEfapOutWrapper, + ( headRotData->headRotEnabled ) ? rotatedPos.azimuth : ismInput->previousPos.azimuth, + ( headRotData->headRotEnabled ) ? rotatedPos.elevation : ismInput->previousPos.elevation, + previousPanGains ) ) != IVAS_ERR_OK ) + { + return error; + } - renderBufferChannelLerp( ismInput->base.inputBuffer, 0, currentPanGains, previousPanGains, tmpMcBuffer ); + /* current position gains */ + if ( headRotData->headRotEnabled ) + { + rotateAziEle( ismInput->currentPos.azimuth, ismInput->currentPos.elevation, &azi_rot, &ele_rot, Rmat, 0 ); + rotatedPos.azimuth = (float) azi_rot; + rotatedPos.elevation = (float) ele_rot; + } + if ( ( error = getEfapGains( *ismInput->base.ctx.pEfapOutWrapper, + ( headRotData->headRotEnabled ) ? rotatedPos.azimuth : ismInput->currentPos.azimuth, + ( headRotData->headRotEnabled ) ? rotatedPos.elevation : ismInput->currentPos.elevation, + currentPanGains ) ) != IVAS_ERR_OK ) + { + return error; + } + + for ( i = 0; i < 3; i++ ) + { + mvr2r( Rmat[i], ismInput->rot_mat_prev[i], 3 ); + } + + /* intermediate rendering to 7_1_4 */ + tmpMcBuffer = ismInput->base.inputBuffer; + getAudioConfigNumChannels( IVAS_REND_AUDIO_CONFIG_7_1_4, &tmp ); + tmpMcBuffer.config.numChannels = tmp; + tmpMcBuffer.data = malloc( tmpMcBuffer.config.numSamplesPerChannel * tmpMcBuffer.config.numChannels * sizeof( float ) ); + set_zero( tmpMcBuffer.data, tmpMcBuffer.config.numSamplesPerChannel * tmpMcBuffer.config.numChannels ); + renderBufferChannelLerp( ismInput->base.inputBuffer, 0, currentPanGains, previousPanGains, tmpMcBuffer ); + +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + copyBufferTo2dArray( tmpMcBuffer, tmpRendBuffer ); +#else copyBufferTo2dArray( tmpMcBuffer, tmpCrendBuffer ); +#endif - ivas_rend_crendProcess( + ivas_rend_crendProcess( #ifdef FIX_197_CREND_INTERFACE - ismInput->crendWrapper, + ismInput->crendWrapper, #else &ismInput->crendWrapper, #endif - AUDIO_CONFIG_7_1_4, - AUDIO_CONFIG_BINAURAL_ROOM, + AUDIO_CONFIG_7_1_4, + AUDIO_CONFIG_BINAURAL_ROOM, #ifdef FIX_197_CREND_INTERFACE - NULL, - NULL, - NULL, - NULL, + NULL, + NULL, + NULL, + NULL, #endif +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + tmpRendBuffer, +#else tmpCrendBuffer, - *ismInput->base.ctx.pOutSampleRate ); +#endif + *ismInput->base.ctx.pOutSampleRate ); +#ifndef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + accumulate2dArrayToBuffer( tmpCrendBuffer, &outAudio ); +#endif + free( tmpMcBuffer.data ); +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + } + accumulate2dArrayToBuffer( tmpRendBuffer, &outAudio ); +#else accumulate2dArrayToBuffer( tmpCrendBuffer, &outAudio ); - - free( tmpMcBuffer.data ); - +#endif pop_wmops(); return IVAS_ERR_OK; @@ -4570,6 +4683,9 @@ static ivas_error renderMcToBinaural( &mcInput->customLsInput, mcInput->base.ctx.pHeadRotData, NULL, +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + mcInput->reverb, +#endif mcInput->base.inputBuffer.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) { @@ -4639,59 +4755,108 @@ static ivas_error renderMcToBinauralRoom( const IVAS_REND_AudioConfig outConfig, IVAS_REND_AudioBuffer outAudio ) { +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + int8_t headRotEnabled; + float tmpRendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + IVAS_REND_AudioConfig inConfig; +#else float tmpCrendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; +#endif ivas_error error; IVAS_REND_AudioBuffer tmpRotBuffer; push_wmops( "renderMcToBinauralRoom" ); - /* apply rotation */ - if ( mcInput->base.ctx.pHeadRotData->headRotEnabled ) - { - tmpRotBuffer = mcInput->base.inputBuffer; - tmpRotBuffer.data = malloc( tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels * sizeof( float ) ); - set_zero( tmpRotBuffer.data, tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels ); +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + headRotEnabled = mcInput->base.ctx.pHeadRotData->headRotEnabled; + inConfig = mcInput->base.inConfig; - rotateFrameMc( mcInput->base.inputBuffer, - mcInput->base.inConfig, - mcInput->customLsInput, - mcInput->base.ctx.pHeadRotData, - mcInput->rot_gains_prev, - mcInput->efapInWrapper.hEfap, - tmpRotBuffer ); + if ( ( mcInput->reverb != NULL && mcInput->reverb->pConfig.roomAcoustics.use_brir == 0 && mcInput->reverb->pConfig.roomAcoustics.late_reverb_on == 1 ) && ( ( inConfig == IVAS_REND_AUDIO_CONFIG_LS_CUSTOM ) || ( headRotEnabled && ( inConfig == IVAS_REND_AUDIO_CONFIG_5_1 || inConfig == IVAS_REND_AUDIO_CONFIG_7_1 ) ) ) ) + { + copyBufferTo2dArray( mcInput->base.inputBuffer, tmpRendBuffer ); - copyBufferTo2dArray( tmpRotBuffer, tmpCrendBuffer ); - free( tmpRotBuffer.data ); + if ( ( error = ivas_td_binaural_renderer_ext( &mcInput->tdRendWrapper, + mcInput->base.inConfig, + &mcInput->customLsInput, + mcInput->base.ctx.pHeadRotData, + NULL, + mcInput->reverb, + mcInput->base.inputBuffer.config.numSamplesPerChannel, + tmpRendBuffer ) ) != IVAS_ERR_OK ) + { + return error; + } } else { + + /* apply rotation */ + if ( headRotEnabled ) +#else + /* apply rotation */ + if ( mcInput->base.ctx.pHeadRotData->headRotEnabled ) +#endif + { + tmpRotBuffer = mcInput->base.inputBuffer; + tmpRotBuffer.data = malloc( tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels * sizeof( float ) ); + set_zero( tmpRotBuffer.data, tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels ); + + rotateFrameMc( mcInput->base.inputBuffer, + mcInput->base.inConfig, + mcInput->customLsInput, + mcInput->base.ctx.pHeadRotData, + mcInput->rot_gains_prev, + mcInput->efapInWrapper.hEfap, + tmpRotBuffer ); + +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + copyBufferTo2dArray( tmpRotBuffer, tmpRendBuffer ); +#else + copyBufferTo2dArray( tmpRotBuffer, tmpCrendBuffer ); +#endif + free( tmpRotBuffer.data ); + } + else + { +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + copyBufferTo2dArray( mcInput->base.inputBuffer, tmpRendBuffer ); +#else copyBufferTo2dArray( mcInput->base.inputBuffer, tmpCrendBuffer ); - } +#endif + } - /* call CREND */ - if ( ( error = ivas_rend_crendProcess( + /* call CREND */ + if ( ( error = ivas_rend_crendProcess( #ifdef FIX_197_CREND_INTERFACE - mcInput->crendWrapper, - getIvasAudioConfigFromRendAudioConfig( mcInput->base.inConfig ), - getIvasAudioConfigFromRendAudioConfig( outConfig ), - NULL, - NULL, - NULL, - NULL, + mcInput->crendWrapper, + getIvasAudioConfigFromRendAudioConfig( mcInput->base.inConfig ), + getIvasAudioConfigFromRendAudioConfig( outConfig ), + NULL, + NULL, + NULL, + NULL, #else &mcInput->crendWrapper, rendAudioConfigToIvasAudioConfig( mcInput->base.inConfig ), rendAudioConfigToIvasAudioConfig( outConfig ), #endif +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND + tmpRendBuffer, +#else tmpCrendBuffer, - *mcInput->base.ctx.pOutSampleRate ) ) != IVAS_ERR_OK ) - { - return error; +#endif + *mcInput->base.ctx.pOutSampleRate ) ) != IVAS_ERR_OK ) + { + return error; + } +#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND } + accumulate2dArrayToBuffer( tmpRendBuffer, &outAudio ); +#else accumulate2dArrayToBuffer( tmpCrendBuffer, &outAudio ); - +#endif /* TODO tmu : needs delay compensation */ renderLfeToBinaural( mcInput, outAudio ); -- GitLab From b24077f7e42fd2948a49088e7adfdc6c13832c3d Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Mon, 6 Mar 2023 15:06:18 +0100 Subject: [PATCH 2/2] [fix] add reverb only if late_reverb_on flag is 1 --- lib_rend/ivas_objectRenderer.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib_rend/ivas_objectRenderer.c b/lib_rend/ivas_objectRenderer.c index 69a8c88219..57390d3f91 100644 --- a/lib_rend/ivas_objectRenderer.c +++ b/lib_rend/ivas_objectRenderer.c @@ -323,23 +323,23 @@ void ivas_td_binaural_renderer_unwrap( #ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND - if ( hReverb != NULL ) + if ( ( hReverb != NULL ) && ( hReverb->pConfig.roomAcoustics.late_reverb_on ) ) + { + /* add reverb to rendered signals */ + v_add( reverb_signal[0], output[0], output[0], output_frame ); + v_add( reverb_signal[1], output[1], output[1], output_frame ); + } #else if ( hRenderConfig != NULL ) /* Renderer Configuration not enabled in TD standalone renderer */ -#endif { - -#ifdef FIX_330_ENABLE_TD_RENDERER_REVERB_REND - if ( hReverb != NULL ) -#else if ( hRenderConfig->roomAcoustics.late_reverb_on ) -#endif { /* add reverb to rendered signals */ v_add( reverb_signal[0], output[0], output[0], output_frame ); v_add( reverb_signal[1], output[1], output[1], output_frame ); } } +#endif return; } -- GitLab