From 838731d2f22e96a5113b49bcf6c3cdc9e2793805 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 10 May 2023 10:06:50 +0200 Subject: [PATCH 01/18] Add fixes under FIX_356_ISM_METADATA_SYNC --- apps/decoder.c | 5 +- lib_com/options.h | 2 +- lib_dec/ivas_objectRenderer_internal.c | 4 + lib_dec/ivas_stat_dec.h | 3 + lib_dec/lib_dec.c | 13 +- lib_dec/lib_dec.h | 5 + lib_rend/ivas_objectRenderer.c | 658 +++++++++++++------------ lib_rend/ivas_prot_rend.h | 6 + lib_rend/lib_rend.c | 38 ++ 9 files changed, 417 insertions(+), 317 deletions(-) diff --git a/apps/decoder.c b/apps/decoder.c index 22dc235fff..34ae45d396 100644 --- a/apps/decoder.c +++ b/apps/decoder.c @@ -354,8 +354,11 @@ int main( /*------------------------------------------------------------------------------------------* * Configure the decoder *------------------------------------------------------------------------------------------*/ - +#ifdef FIX_356_ISM_METADATA_SYNC + if ( ( error = IVAS_DEC_Configure( hIvasDec, arg.output_Fs, arg.outputFormat, arg.customLsOutputEnabled, arg.hrtfReaderEnabled, arg.enableHeadRotation, arg.renderConfigEnabled, arg.delayCompensationEnabled ) ) != IVAS_ERR_OK ) +#else if ( ( error = IVAS_DEC_Configure( hIvasDec, arg.output_Fs, arg.outputFormat, arg.customLsOutputEnabled, arg.hrtfReaderEnabled, arg.enableHeadRotation, arg.renderConfigEnabled ) ) != IVAS_ERR_OK ) +#endif { fprintf( stderr, "\nConfigure failed: %s\n\n", IVAS_DEC_GetErrorMessage( error ) ); goto cleanup; diff --git a/lib_com/options.h b/lib_com/options.h index 78b765d30e..849e70bec0 100644 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -179,7 +179,7 @@ #define FIX_355_REFACTOR_PARAMBIN_TO_5MS /* Nokia: Fixes issue 355 by refactoring parametric binauralizer code to 5 ms mode */ #define FIX_411_EVS_BE_TESTS_ON_WINDOWS_FAILING /* Eri: Fix incorrect use of stack variable used for channel aware config file */ #define COMBINED_FORMAT_SIGNALING /* VA: Introduce a signaling bit for combined format coding */ - +#define FIX_356_ISM_METADATA_SYNC /* Eri: issue 356: Metadata out-of-synch for -no_delay_comp */ #define FIX_440_PARAM_ISM_DIR_NOISE /* FhG: Issue 440: Fix directional background noise becoming diffuse in ParamISM */ diff --git a/lib_dec/ivas_objectRenderer_internal.c b/lib_dec/ivas_objectRenderer_internal.c index f1c1fc04f2..1dd3a1b1c8 100644 --- a/lib_dec/ivas_objectRenderer_internal.c +++ b/lib_dec/ivas_objectRenderer_internal.c @@ -75,6 +75,10 @@ ivas_error ivas_td_binaural_renderer( st_ivas->hReverb, st_ivas->transport_config, st_ivas->hBinRendererTd, st_ivas->nchan_transport, LFE_CHANNEL, st_ivas->ivas_format, +#ifdef FIX_356_ISM_METADATA_SYNC + st_ivas->hIsmMetaData, st_ivas->hDecoderConfig->Opt_delay_comp, st_ivas->hDecoderConfig->Opt_Headrotation, ( st_ivas->hHeadTrackData != NULL ) ? st_ivas->hHeadTrackData->Quaternions : NULL, +#else st_ivas->hIsmMetaData, st_ivas->hDecoderConfig->Opt_Headrotation, ( st_ivas->hHeadTrackData != NULL ) ? st_ivas->hHeadTrackData->Quaternions : NULL, +#endif ( st_ivas->hHeadTrackData != NULL ) ? st_ivas->hHeadTrackData->Pos : NULL, output, output_frame ); } diff --git a/lib_dec/ivas_stat_dec.h b/lib_dec/ivas_stat_dec.h index ad453cd5bd..39e13e771a 100644 --- a/lib_dec/ivas_stat_dec.h +++ b/lib_dec/ivas_stat_dec.h @@ -1145,6 +1145,9 @@ typedef struct decoder_config_structure int16_t orientation_tracking; /* indicates orientation tracking type */ float no_diegetic_pan; int16_t Opt_AMR_WB; /* flag indicating AMR-WB IO mode */ +#ifdef FIX_356_ISM_METADATA_SYNC + int16_t Opt_delay_comp; /* flag indicating delay compensation active */ +#endif /* temp. development parameters */ #ifdef DEBUGGING diff --git a/lib_dec/lib_dec.c b/lib_dec/lib_dec.c index ee315bf399..5791fda0c6 100644 --- a/lib_dec/lib_dec.c +++ b/lib_dec/lib_dec.c @@ -226,6 +226,9 @@ static void init_decoder_config( hDecoderConfig->Opt_RendConfigCustom = 0; hDecoderConfig->orientation_tracking = orientation_tracking; hDecoderConfig->no_diegetic_pan = no_diegetic_pan; +#ifdef FIX_356_ISM_METADATA_SYNC + hDecoderConfig->Opt_delay_comp = 0; +#endif return; } @@ -388,7 +391,12 @@ ivas_error IVAS_DEC_Configure( const int16_t customLsOutputEnabled, /* i : enable custom loudspeaker setup handle */ const int16_t hrtfReaderEnabled, /* i : enable HRTF binary file input */ const int16_t enableHeadRotation, /* i : enable head rotation for binaural output */ - const int16_t renderConfigEnabled /* i : enable Renderer config. file for binaural output */ +#ifdef FIX_356_ISM_METADATA_SYNC + const int16_t renderConfigEnabled, /* i : enable Renderer config. file for binaural output */ + const int16_t delayCompensationEnabled /* i : enable delay compensation */ +#else + const int16_t renderConfigEnabled /* i : enable Renderer config. file for binaural output */ +#endif ) { Decoder_Struct *st_ivas; @@ -438,6 +446,9 @@ ivas_error IVAS_DEC_Configure( hDecoderConfig->Opt_Headrotation = enableHeadRotation; hDecoderConfig->Opt_HRTF_binary = hrtfReaderEnabled; hDecoderConfig->Opt_RendConfigCustom = renderConfigEnabled; +#ifdef FIX_356_ISM_METADATA_SYNC + hDecoderConfig->Opt_delay_comp = delayCompensationEnabled; +#endif /* Set decoder parameters to initial values */ if ( ( error = ivas_init_decoder_front( st_ivas ) ) != IVAS_ERR_OK ) diff --git a/lib_dec/lib_dec.h b/lib_dec/lib_dec.h index 17af564295..16aa8a66dd 100644 --- a/lib_dec/lib_dec.h +++ b/lib_dec/lib_dec.h @@ -130,7 +130,12 @@ ivas_error IVAS_DEC_Configure( const int16_t customLsOutputEnabled, /* i : enable custom loudspeaker setup handle */ const int16_t hrtfReaderEnabled, /* i : enable HRTF binary file input */ const int16_t enableHeadRotation, /* i : enable head rotation for binaural output */ +#ifdef FIX_356_ISM_METADATA_SYNC + const int16_t renderConfigEnabled, /* i : enable Renderer config. file for binaural output */ + const int16_t delayCompensationEnabled /* i : enable delay compensation */ +#else const int16_t renderConfigEnabled /* i : enable Renderer config. file for binaural output */ +#endif ); void IVAS_DEC_Close( diff --git a/lib_rend/ivas_objectRenderer.c b/lib_rend/ivas_objectRenderer.c index 0618534448..a5d3d30863 100644 --- a/lib_rend/ivas_objectRenderer.c +++ b/lib_rend/ivas_objectRenderer.c @@ -256,398 +256,428 @@ ivas_error ivas_td_binaural_renderer_unwrap( const int16_t lfe_idx, /* i : LFE channel index */ const IVAS_FORMAT ivas_format, /* i : IVAS format */ ISM_METADATA_HANDLE *hIsmMetaData, /* i : ISM metadata handle */ - const int16_t Opt_Headrotation, /* i : Head rotation flag */ - const IVAS_QUATERNION *Quaternions, /* i : Head tracking data per subframe */ - const IVAS_VECTOR3 *Pos, /* i : Listener position data per subframe */ - float output[][L_FRAME48k], /* i/o: SCE channels / Binaural synthesis */ - const int16_t output_frame /* i : output frame length */ +#ifdef FIX_356_ISM_METADATA_SYNC + const int16_t Opt_delay_comp, /* i : Delay compensation flag */ +#endif + const int16_t Opt_Headrotation, /* i : Head rotation flag */ + const IVAS_QUATERNION *Quaternions, /* i : Head tracking data per subframe */ + const IVAS_VECTOR3 *Pos, /* i : Listener position data per subframe */ + float output[][L_FRAME48k], /* i/o: SCE channels / Binaural synthesis */ + const int16_t output_frame /* i : output frame length */ ) { int16_t subframe_length; int16_t subframe_idx; float reverb_signal[BINAURAL_CHANNELS][L_FRAME48k]; ivas_error error; +#ifdef FIX_356_ISM_METADATA_SYNC + int16_t subframe_update; + if ( Opt_delay_comp ) + { + subframe_update = 0; + } + else + { + subframe_update = 2; /* To be verified */ + } +#endif subframe_length = output_frame / MAX_PARAM_SPATIAL_SUBFRAMES; +#ifndef FIX_356_ISM_METADATA_SYNC /* Update object position(s) */ - TDREND_Update_object_positions( hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, output ); + TDREND_Update_object_positions( hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, output ); +#endif + for ( subframe_idx = 0; subframe_idx < MAX_PARAM_SPATIAL_SUBFRAMES; subframe_idx++ ) + { +#ifdef FIX_356_ISM_METADATA_SYNC + if ( subframe_idx == subframe_update ) + { + /* Update object position(s) */ + TDREND_Update_object_positions( hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, output ); + } +#endif - for ( subframe_idx = 0; subframe_idx < MAX_PARAM_SPATIAL_SUBFRAMES; subframe_idx++ ) - { - /* Update the listener's location/orientation */ - TDREND_Update_listener_orientation( hBinRendererTd, Opt_Headrotation, ( Quaternions != NULL ) ? &Quaternions[subframe_idx] : NULL, ( Pos != NULL ) ? &Pos[subframe_idx] : NULL ); + /* Update the listener's location/orientation */ + TDREND_Update_listener_orientation( hBinRendererTd, Opt_Headrotation, ( Quaternions != NULL ) ? &Quaternions[subframe_idx] : NULL, ( Pos != NULL ) ? &Pos[subframe_idx] : NULL ); - if ( hReverb != NULL && hReverb->pConfig.roomAcoustics.late_reverb_on ) - { - if ( ( error = ivas_reverb_process( hReverb, transport_config, 0, output, reverb_signal, subframe_idx ) ) != IVAS_ERR_OK ) + if ( hReverb != NULL && hReverb->pConfig.roomAcoustics.late_reverb_on ) + { + if ( ( error = ivas_reverb_process( hReverb, transport_config, 0, output, reverb_signal, subframe_idx ) ) != IVAS_ERR_OK ) + { + return error; + } + } + + /* Render subframe */ + if ( ( error = TDREND_GetMix( hBinRendererTd, output, subframe_length, subframe_idx ) ) != IVAS_ERR_OK ) { return error; } } - /* Render subframe */ - if ( ( error = TDREND_GetMix( hBinRendererTd, output, subframe_length, subframe_idx ) ) != IVAS_ERR_OK ) + + if ( hReverb != NULL && hReverb->pConfig.roomAcoustics.late_reverb_on ) { - return error; + /* add reverb to rendered signals */ + v_add( reverb_signal[0], output[0], output[0], output_frame ); + v_add( reverb_signal[1], output[1], output[1], output_frame ); } + + return IVAS_ERR_OK; } - if ( hReverb != NULL && hReverb->pConfig.roomAcoustics.late_reverb_on ) + /*---------------------------------------------------------------------* + * TDREND_GetMix() + * + * Render one 5 ms subframe from the mixer + *---------------------------------------------------------------------*/ + + ivas_error TDREND_GetMix( + BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD renderer handle */ + float output[][L_FRAME48k], /* i/o: ISM object synth / rendered output in 0,1 */ + const int16_t subframe_length, /* i/o: subframe length */ + const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ + ) { - /* add reverb to rendered signals */ - v_add( reverb_signal[0], output[0], output[0], output_frame ); - v_add( reverb_signal[1], output[1], output[1], output_frame ); - } + int16_t i; + TDREND_SRC_t *Src_p; + TDREND_SRC_SPATIAL_t *SrcSpatial_p; + TDREND_SRC_REND_t *SrcRend_p; + ivas_error error; + float output_buf[2][L_SPATIAL_SUBFR_48k]; /* Temp buffer for left/right rendered signal */ + float hrf_left_delta[SFX_SPAT_BIN_MAX_FILTER_LENGTH]; + float hrf_right_delta[SFX_SPAT_BIN_MAX_FILTER_LENGTH]; + int16_t intp_count; + error = IVAS_ERR_OK; + + /* Clear the output buffer to accumulate rendered sources */ + set_f( output_buf[0], 0.0f, subframe_length ); + set_f( output_buf[1], 0.0f, subframe_length ); + + /* Clear interpolation buffers and counter */ + set_f( hrf_left_delta, 0.0f, SFX_SPAT_BIN_MAX_FILTER_LENGTH ); + set_f( hrf_right_delta, 0.0f, SFX_SPAT_BIN_MAX_FILTER_LENGTH ); + intp_count = 0; + + /* Create the mix */ + /* Loop through the source list and render each source */ + for ( i = 0; i < hBinRendererTd->NumOfSrcs; i++ ) + { + Src_p = hBinRendererTd->Sources[i]; + SrcSpatial_p = Src_p->SrcSpatial_p; + SrcRend_p = Src_p->SrcRend_p; - return IVAS_ERR_OK; -} + /* Update rendering params if needed */ + if ( hBinRendererTd->Listener_p->PoseUpdated || SrcSpatial_p->Updated ) + { + TDREND_SRC_REND_UpdateFiltersFromSpatialParams( hBinRendererTd, SrcRend_p, SrcSpatial_p, Src_p->hrf_left_prev, + Src_p->hrf_right_prev, hrf_left_delta, hrf_right_delta, &intp_count, &Src_p->filterlength, &Src_p->itd, &Src_p->Gain, Src_p, subframe_idx ); + } + /* Render source if needed */ + if ( ( SrcRend_p->InputAvailable == TRUE ) && ( SrcRend_p->PlayStatus == TDREND_PLAYSTATUS_PLAYING ) ) + { + error = TDREND_REND_RenderSourceHRFilt( Src_p, hrf_left_delta, hrf_right_delta, intp_count, output_buf, subframe_length ); + } + } -/*---------------------------------------------------------------------* - * TDREND_GetMix() - * - * Render one 5 ms subframe from the mixer - *---------------------------------------------------------------------*/ + /* Populate output variable */ + mvr2r( output_buf[0], output[0] + subframe_idx * subframe_length, subframe_length ); /* Left */ + mvr2r( output_buf[1], output[1] + subframe_idx * subframe_length, subframe_length ); /* Right */ -ivas_error TDREND_GetMix( - BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD renderer handle */ - float output[][L_FRAME48k], /* i/o: ISM object synth / rendered output in 0,1 */ - const int16_t subframe_length, /* i/o: subframe length */ - const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ -) -{ - int16_t i; - TDREND_SRC_t *Src_p; - TDREND_SRC_SPATIAL_t *SrcSpatial_p; - TDREND_SRC_REND_t *SrcRend_p; - ivas_error error; - float output_buf[2][L_SPATIAL_SUBFR_48k]; /* Temp buffer for left/right rendered signal */ - float hrf_left_delta[SFX_SPAT_BIN_MAX_FILTER_LENGTH]; - float hrf_right_delta[SFX_SPAT_BIN_MAX_FILTER_LENGTH]; - int16_t intp_count; - error = IVAS_ERR_OK; + /* Clear the PoseUpdated and Source position update flags */ + TDREND_Clear_Update_flags( hBinRendererTd ); + + return error; + } - /* Clear the output buffer to accumulate rendered sources */ - set_f( output_buf[0], 0.0f, subframe_length ); - set_f( output_buf[1], 0.0f, subframe_length ); - /* Clear interpolation buffers and counter */ - set_f( hrf_left_delta, 0.0f, SFX_SPAT_BIN_MAX_FILTER_LENGTH ); - set_f( hrf_right_delta, 0.0f, SFX_SPAT_BIN_MAX_FILTER_LENGTH ); - intp_count = 0; + /*---------------------------------------------------------------------* + * TDREND_Clear_Update_flags() + * + * Initializes the audio mixer module + *---------------------------------------------------------------------*/ - /* Create the mix */ - /* Loop through the source list and render each source */ - for ( i = 0; i < hBinRendererTd->NumOfSrcs; i++ ) + static void TDREND_Clear_Update_flags( + BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd /* i/o: TD renderer handle */ + ) { - Src_p = hBinRendererTd->Sources[i]; - SrcSpatial_p = Src_p->SrcSpatial_p; - SrcRend_p = Src_p->SrcRend_p; + int16_t i; - /* Update rendering params if needed */ - if ( hBinRendererTd->Listener_p->PoseUpdated || SrcSpatial_p->Updated ) - { - TDREND_SRC_REND_UpdateFiltersFromSpatialParams( hBinRendererTd, SrcRend_p, SrcSpatial_p, Src_p->hrf_left_prev, - Src_p->hrf_right_prev, hrf_left_delta, hrf_right_delta, &intp_count, &Src_p->filterlength, &Src_p->itd, &Src_p->Gain, Src_p, subframe_idx ); - } + hBinRendererTd->Listener_p->PoseUpdated = FALSE; - /* Render source if needed */ - if ( ( SrcRend_p->InputAvailable == TRUE ) && ( SrcRend_p->PlayStatus == TDREND_PLAYSTATUS_PLAYING ) ) + for ( i = 0; i < hBinRendererTd->NumOfSrcs; i++ ) { - error = TDREND_REND_RenderSourceHRFilt( Src_p, hrf_left_delta, hrf_right_delta, intp_count, output_buf, subframe_length ); + hBinRendererTd->Sources[i]->SrcSpatial_p->Updated = FALSE; } - } - - /* Populate output variable */ - mvr2r( output_buf[0], output[0] + subframe_idx * subframe_length, subframe_length ); /* Left */ - mvr2r( output_buf[1], output[1] + subframe_idx * subframe_length, subframe_length ); /* Right */ - /* Clear the PoseUpdated and Source position update flags */ - TDREND_Clear_Update_flags( hBinRendererTd ); + return; + } - return error; -} + /*---------------------------------------------------------------------* + * TDREND_Update_object_positions() + * + * Update object position(s) + *---------------------------------------------------------------------*/ + + void TDREND_Update_object_positions( + BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o : TD Renderer handle */ + const int16_t num_src, /* i : number of sources to render */ + const int16_t lfe_idx, /* i : Input LFE index */ + const IVAS_FORMAT in_format, /* i : Format of input sources */ + const ISM_METADATA_HANDLE *hIsmMetaData, /* i : Input metadata for ISM objects */ + float output[][L_FRAME48k] /* i/o: SCE/MC channels */ + ) + { + TDREND_DirAtten_t *DirAtten_p; + int16_t nS; + float Pos[3]; + float Dir[3]; + int16_t c_indx; -/*---------------------------------------------------------------------* - * TDREND_Clear_Update_flags() - * - * Initializes the audio mixer module - *---------------------------------------------------------------------*/ + DirAtten_p = hBinRendererTd->DirAtten_p; -static void TDREND_Clear_Update_flags( - BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd /* i/o: TD renderer handle */ -) -{ - int16_t i; + /* For each source, write the frame data to the source object*/ + c_indx = 0; + for ( nS = 0; nS < num_src; nS++ ) + { + if ( !( in_format == MC_FORMAT && nS == lfe_idx ) ) /* Skip LFE for MC */ + { + hBinRendererTd->Sources[c_indx]->InputFrame_p = output[nS]; + hBinRendererTd->Sources[c_indx]->SrcRend_p->InputAvailable = TRUE; + c_indx++; + } - hBinRendererTd->Listener_p->PoseUpdated = FALSE; + if ( in_format == ISM_FORMAT ) + { - for ( i = 0; i < hBinRendererTd->NumOfSrcs; i++ ) - { - hBinRendererTd->Sources[i]->SrcSpatial_p->Updated = FALSE; - } + /* Update the source positions */ + /* Source position and direction */ + angles_to_vec( hIsmMetaData[nS]->radius, hIsmMetaData[nS]->azimuth, hIsmMetaData[nS]->elevation, Pos ); + angles_to_vec( 1.0f, hIsmMetaData[nS]->yaw, hIsmMetaData[nS]->pitch, Dir ); + /* Source directivity info */ + DirAtten_p->ConeInnerAngle = 360.0f; + DirAtten_p->ConeOuterAngle = 360.0f; + DirAtten_p->ConeOuterGain = 1.0f; - return; -} + TDREND_MIX_SRC_SetPos( hBinRendererTd, nS, Pos ); + TDREND_MIX_SRC_SetDirAtten( hBinRendererTd, nS, DirAtten_p ); + TDREND_MIX_SRC_SetPlayState( hBinRendererTd, nS, TDREND_PLAYSTATUS_PLAYING ); + TDREND_MIX_SRC_SetDir( hBinRendererTd, nS, Dir ); + } + } -/*---------------------------------------------------------------------* - * TDREND_Update_object_positions() - * - * Update object position(s) - *---------------------------------------------------------------------*/ + return; + } -void TDREND_Update_object_positions( - BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o : TD Renderer handle */ - const int16_t num_src, /* i : number of sources to render */ - const int16_t lfe_idx, /* i : Input LFE index */ - const IVAS_FORMAT in_format, /* i : Format of input sources */ - const ISM_METADATA_HANDLE *hIsmMetaData, /* i : Input metadata for ISM objects */ - float output[][L_FRAME48k] /* i/o: SCE/MC channels */ -) -{ - TDREND_DirAtten_t *DirAtten_p; - int16_t nS; - float Pos[3]; - float Dir[3]; - int16_t c_indx; - DirAtten_p = hBinRendererTd->DirAtten_p; + /*---------------------------------------------------------------------* + * TDREND_Update_listener_orientation() + * + * Update listener orientation (s) + *---------------------------------------------------------------------*/ - /* For each source, write the frame data to the source object*/ - c_indx = 0; - for ( nS = 0; nS < num_src; nS++ ) + void TDREND_Update_listener_orientation( + BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD Renderer handle */ + const int16_t headRotEnabled, /* i : Headrotation flag */ + const IVAS_QUATERNION *headPosition, /* i : Listener orientation */ + const IVAS_VECTOR3 *Pos /* i : Listener Position */ + ) { - if ( !( in_format == MC_FORMAT && nS == lfe_idx ) ) /* Skip LFE for MC */ + float FrontVec[3]; + float UpVec[3]; + float Rmat[3][3]; + float Pos_p[3]; + + if ( headRotEnabled ) { - hBinRendererTd->Sources[c_indx]->InputFrame_p = output[nS]; - hBinRendererTd->Sources[c_indx]->SrcRend_p->InputAvailable = TRUE; - c_indx++; + /* Obtain head rotation matrix */ + QuatToRotMat( *headPosition, Rmat ); + /* Apply rotation matrix to looking vector [1;0;0] */ + FrontVec[0] = Rmat[0][0]; + FrontVec[1] = Rmat[0][1]; + FrontVec[2] = Rmat[0][2]; + /* Apply rotation matrix to up vector [0;0;1] */ + UpVec[0] = Rmat[2][0]; + UpVec[1] = Rmat[2][1]; + UpVec[2] = Rmat[2][2]; + /* Input position */ + Pos_p[0] = ( *Pos ).x; + Pos_p[1] = ( *Pos ).y; + Pos_p[2] = ( *Pos ).z; } - - if ( in_format == ISM_FORMAT ) + else { - - /* Update the source positions */ - /* Source position and direction */ - angles_to_vec( hIsmMetaData[nS]->radius, hIsmMetaData[nS]->azimuth, hIsmMetaData[nS]->elevation, Pos ); - angles_to_vec( 1.0f, hIsmMetaData[nS]->yaw, hIsmMetaData[nS]->pitch, Dir ); - /* Source directivity info */ - DirAtten_p->ConeInnerAngle = 360.0f; - DirAtten_p->ConeOuterAngle = 360.0f; - DirAtten_p->ConeOuterGain = 1.0f; - - TDREND_MIX_SRC_SetPos( hBinRendererTd, nS, Pos ); - TDREND_MIX_SRC_SetDirAtten( hBinRendererTd, nS, DirAtten_p ); - TDREND_MIX_SRC_SetPlayState( hBinRendererTd, nS, TDREND_PLAYSTATUS_PLAYING ); - - TDREND_MIX_SRC_SetDir( hBinRendererTd, nS, Dir ); + /* Oriented with looking vector along the x axis */ + FrontVec[0] = 1.0f; + FrontVec[1] = 0.0f; + FrontVec[2] = 0.0f; + /* Oriented with up vector along the z axis */ + UpVec[0] = 0.0f; + UpVec[1] = 0.0f; + UpVec[2] = 1.0f; + /* Listener at the origin */ + Pos_p[0] = 0.0f; + Pos_p[1] = 0.0f; + Pos_p[2] = 0.0f; } - } - - return; -} - -/*---------------------------------------------------------------------* - * TDREND_Update_listener_orientation() - * - * Update listener orientation (s) - *---------------------------------------------------------------------*/ - -void TDREND_Update_listener_orientation( - BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD Renderer handle */ - const int16_t headRotEnabled, /* i : Headrotation flag */ - const IVAS_QUATERNION *headPosition, /* i : Listener orientation */ - const IVAS_VECTOR3 *Pos /* i : Listener Position */ -) -{ - float FrontVec[3]; - float UpVec[3]; - float Rmat[3][3]; - float Pos_p[3]; + /* Set the listener position and orientation:*/ + TDREND_MIX_LIST_SetPos( hBinRendererTd, Pos_p ); + TDREND_MIX_LIST_SetOrient( hBinRendererTd, FrontVec, UpVec ); - if ( headRotEnabled ) - { - /* Obtain head rotation matrix */ - QuatToRotMat( *headPosition, Rmat ); - /* Apply rotation matrix to looking vector [1;0;0] */ - FrontVec[0] = Rmat[0][0]; - FrontVec[1] = Rmat[0][1]; - FrontVec[2] = Rmat[0][2]; - /* Apply rotation matrix to up vector [0;0;1] */ - UpVec[0] = Rmat[2][0]; - UpVec[1] = Rmat[2][1]; - UpVec[2] = Rmat[2][2]; - /* Input position */ - Pos_p[0] = ( *Pos ).x; - Pos_p[1] = ( *Pos ).y; - Pos_p[2] = ( *Pos ).z; - } - else - { - /* Oriented with looking vector along the x axis */ - FrontVec[0] = 1.0f; - FrontVec[1] = 0.0f; - FrontVec[2] = 0.0f; - /* Oriented with up vector along the z axis */ - UpVec[0] = 0.0f; - UpVec[1] = 0.0f; - UpVec[2] = 1.0f; - /* Listener at the origin */ - Pos_p[0] = 0.0f; - Pos_p[1] = 0.0f; - Pos_p[2] = 0.0f; + return; } - /* Set the listener position and orientation:*/ - TDREND_MIX_LIST_SetPos( hBinRendererTd, Pos_p ); - TDREND_MIX_LIST_SetOrient( hBinRendererTd, FrontVec, UpVec ); - return; -} + /*---------------------------------------------------------------------* + * ivas_td_binaural_open_ext() + * + * + *---------------------------------------------------------------------*/ + ivas_error ivas_td_binaural_open_ext( + TDREND_WRAPPER * pTDRend, + IVAS_REND_AudioConfig inConfig, + RENDER_CONFIG_DATA * hRendCfg, /* i : Renderer configuration */ + LSSETUP_CUSTOM_STRUCT * customLsInput, + const int32_t outFs ) + { + int16_t nchan_transport; + AUDIO_CONFIG transport_config; + IVAS_FORMAT ivas_format; + IVAS_OUTPUT_SETUP hTransSetup; + ivas_error error; + float *directivity = NULL; -/*---------------------------------------------------------------------* - * ivas_td_binaural_open_ext() - * - * - *---------------------------------------------------------------------*/ + if ( inConfig != IVAS_REND_AUDIO_CONFIG_LS_CUSTOM ) + { + if ( ( error = getAudioConfigNumChannels( inConfig, &nchan_transport ) ) != IVAS_ERR_OK ) + { + return error; + } + } + else + { + nchan_transport = customLsInput->num_spk; + } -ivas_error ivas_td_binaural_open_ext( - TDREND_WRAPPER *pTDRend, - IVAS_REND_AudioConfig inConfig, - RENDER_CONFIG_DATA *hRendCfg, /* i : Renderer configuration */ - LSSETUP_CUSTOM_STRUCT *customLsInput, - const int32_t outFs ) -{ - int16_t nchan_transport; - AUDIO_CONFIG transport_config; - IVAS_FORMAT ivas_format; - IVAS_OUTPUT_SETUP hTransSetup; - ivas_error error; - float *directivity = NULL; + transport_config = getIvasAudioConfigFromRendAudioConfig( inConfig ); + ivas_format = ( getAudioConfigType( inConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) ? MC_FORMAT : ISM_FORMAT; + hTransSetup.ls_azimuth = customLsInput->ls_azimuth; + hTransSetup.ls_elevation = customLsInput->ls_elevation; - if ( inConfig != IVAS_REND_AUDIO_CONFIG_LS_CUSTOM ) - { - if ( ( error = getAudioConfigNumChannels( inConfig, &nchan_transport ) ) != IVAS_ERR_OK ) + if ( NULL != hRendCfg ) { - return error; + directivity = hRendCfg->directivity; } - } - else - { - nchan_transport = customLsInput->num_spk; - } - transport_config = getIvasAudioConfigFromRendAudioConfig( inConfig ); - ivas_format = ( getAudioConfigType( inConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) ? MC_FORMAT : ISM_FORMAT; - hTransSetup.ls_azimuth = customLsInput->ls_azimuth; - hTransSetup.ls_elevation = customLsInput->ls_elevation; - - if ( NULL != hRendCfg ) - { - directivity = hRendCfg->directivity; + return ivas_td_binaural_open_unwrap( &pTDRend->hHrtfTD, outFs, nchan_transport, ivas_format, transport_config, directivity, hTransSetup, &pTDRend->hBinRendererTd, &pTDRend->binaural_latency_ns ); } - return ivas_td_binaural_open_unwrap( &pTDRend->hHrtfTD, outFs, nchan_transport, ivas_format, transport_config, directivity, hTransSetup, &pTDRend->hBinRendererTd, &pTDRend->binaural_latency_ns ); -} - - -/*---------------------------------------------------------------------* - * ivas_td_binaural_renderer_ext() - * - * Receives the current frames for the object streams, updates metadata - * and renders the current frame. - *---------------------------------------------------------------------*/ - -ivas_error ivas_td_binaural_renderer_ext( - const TDREND_WRAPPER *pTDRend, /* i : TD Renderer wrapper structure */ - const IVAS_REND_AudioConfig inConfig, /* i : Input audio configuration */ - const LSSETUP_CUSTOM_STRUCT *customLsInput, /* i : Input custom loudspeaker layout */ - const IVAS_REND_HeadRotData *headRotData, /* i : Input head positions */ - const IVAS_REND_AudioObjectPosition *currentPos, /* i : Object position */ - const REVERB_HANDLE hReverb, /* i : Reverberator handle */ - const int16_t output_frame, /* i : output frame length */ - float output[][L_FRAME48k] /* i/o: SCE channels / Binaural synthesis */ -) -{ - ISM_METADATA_FRAME hIsmMetaDataFrame; - ISM_METADATA_HANDLE hIsmMetaData[1]; - int16_t lfe_idx; - int16_t num_src; - IVAS_FORMAT ivas_format; - IVAS_REND_AudioConfigType inConfigType; - AUDIO_CONFIG transport_config; - ivas_error error; - - push_wmops( "ivas_td_binaural_renderer_ext" ); - inConfigType = getAudioConfigType( inConfig ); - lfe_idx = LFE_CHANNEL; - hIsmMetaData[0] = NULL; - - if ( inConfigType == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) + /*---------------------------------------------------------------------* + * ivas_td_binaural_renderer_ext() + * + * Receives the current frames for the object streams, updates metadata + * and renders the current frame. + *---------------------------------------------------------------------*/ + + ivas_error ivas_td_binaural_renderer_ext( + const TDREND_WRAPPER *pTDRend, /* i : TD Renderer wrapper structure */ + const IVAS_REND_AudioConfig inConfig, /* i : Input audio configuration */ + const LSSETUP_CUSTOM_STRUCT *customLsInput, /* i : Input custom loudspeaker layout */ + const IVAS_REND_HeadRotData *headRotData, /* i : Input head positions */ + const IVAS_REND_AudioObjectPosition *currentPos, /* i : Object position */ + const REVERB_HANDLE hReverb, /* i : Reverberator handle */ +#ifdef FIX_356_ISM_METADATA_SYNC + const int16_t Opt_delay_comp, /* i : Delay compensation flag */ +#endif + const int16_t output_frame, /* i : output frame length */ + float output[][L_FRAME48k] /* i/o: SCE channels / Binaural synthesis */ + ) { - ivas_format = MC_FORMAT; - transport_config = getIvasAudioConfigFromRendAudioConfig( inConfig ); - if ( inConfig != IVAS_REND_AUDIO_CONFIG_LS_CUSTOM ) + ISM_METADATA_FRAME hIsmMetaDataFrame; + ISM_METADATA_HANDLE hIsmMetaData[1]; + int16_t lfe_idx; + int16_t num_src; + IVAS_FORMAT ivas_format; + IVAS_REND_AudioConfigType inConfigType; + AUDIO_CONFIG transport_config; + ivas_error error; + + push_wmops( "ivas_td_binaural_renderer_ext" ); + + inConfigType = getAudioConfigType( inConfig ); + lfe_idx = LFE_CHANNEL; + hIsmMetaData[0] = NULL; + + if ( inConfigType == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) { - if ( ( error = getAudioConfigNumChannels( inConfig, &num_src ) ) != IVAS_ERR_OK ) + ivas_format = MC_FORMAT; + transport_config = getIvasAudioConfigFromRendAudioConfig( inConfig ); + if ( inConfig != IVAS_REND_AUDIO_CONFIG_LS_CUSTOM ) { - return error; + if ( ( error = getAudioConfigNumChannels( inConfig, &num_src ) ) != IVAS_ERR_OK ) + { + return error; + } + } + else + { + lfe_idx = ( customLsInput->num_lfe > 0 ) ? customLsInput->lfe_idx[0] : -1; + num_src = customLsInput->num_spk + customLsInput->num_lfe; } } else { - lfe_idx = ( customLsInput->num_lfe > 0 ) ? customLsInput->lfe_idx[0] : -1; - num_src = customLsInput->num_spk + customLsInput->num_lfe; + ivas_format = ISM_FORMAT; + num_src = 1; + transport_config = AUDIO_CONFIG_ISM1; + hIsmMetaData[0] = &hIsmMetaDataFrame; + hIsmMetaData[0]->azimuth = currentPos->azimuth; + hIsmMetaData[0]->elevation = currentPos->elevation; + hIsmMetaData[0]->yaw = currentPos->yaw; + hIsmMetaData[0]->pitch = currentPos->pitch; + hIsmMetaData[0]->radius = currentPos->radius; } - } - else - { - ivas_format = ISM_FORMAT; - num_src = 1; - transport_config = AUDIO_CONFIG_ISM1; - hIsmMetaData[0] = &hIsmMetaDataFrame; - hIsmMetaData[0]->azimuth = currentPos->azimuth; - hIsmMetaData[0]->elevation = currentPos->elevation; - hIsmMetaData[0]->yaw = currentPos->yaw; - hIsmMetaData[0]->pitch = currentPos->pitch; - hIsmMetaData[0]->radius = currentPos->radius; - } - - if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, - ( headRotData != NULL ) ? headRotData->headPositions : NULL, - ( headRotData != NULL ) ? headRotData->Pos : NULL, output, output_frame ) ) != IVAS_ERR_OK ) - { - return error; - } - pop_wmops(); +#ifdef FIX_356_ISM_METADATA_SYNC + if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, Opt_delay_comp, headRotData->headRotEnabled, +#else + if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, +#endif + ( headRotData != NULL ) ? headRotData->headPositions : NULL, + ( headRotData != NULL ) ? headRotData->Pos : NULL, output, output_frame ) ) != IVAS_ERR_OK ) + { + return error; + } - return IVAS_ERR_OK; -} + pop_wmops(); -/*---------------------------------------------------------------------* - * angles_to_vec() - * - * Convert azimuth and elevation angles to position/orientation vector - *---------------------------------------------------------------------*/ + return IVAS_ERR_OK; + } -static void angles_to_vec( - const float radius, /* i : radius */ - const float azimuth, /* i : Azimuth angle */ - const float elevation, /* i : Elevation angle */ - float *vec /* o : Pos/Dir vector */ -) -{ - vec[0] = radius * cosf( elevation * PI_OVER_180 ) * cosf( azimuth * PI_OVER_180 ); - vec[1] = radius * cosf( elevation * PI_OVER_180 ) * sinf( azimuth * PI_OVER_180 ); - vec[2] = radius * sinf( elevation * PI_OVER_180 ); + /*---------------------------------------------------------------------* + * angles_to_vec() + * + * Convert azimuth and elevation angles to position/orientation vector + *---------------------------------------------------------------------*/ + + static void angles_to_vec( + const float radius, /* i : radius */ + const float azimuth, /* i : Azimuth angle */ + const float elevation, /* i : Elevation angle */ + float *vec /* o : Pos/Dir vector */ + ) + { + vec[0] = radius * cosf( elevation * PI_OVER_180 ) * cosf( azimuth * PI_OVER_180 ); + vec[1] = radius * cosf( elevation * PI_OVER_180 ) * sinf( azimuth * PI_OVER_180 ); + vec[2] = radius * sinf( elevation * PI_OVER_180 ); - return; -} + return; + } diff --git a/lib_rend/ivas_prot_rend.h b/lib_rend/ivas_prot_rend.h index efa2111243..b3232656ae 100644 --- a/lib_rend/ivas_prot_rend.h +++ b/lib_rend/ivas_prot_rend.h @@ -228,6 +228,9 @@ ivas_error ivas_td_binaural_renderer_unwrap( const int16_t lfe_idx, /* i : LFE channel index */ const IVAS_FORMAT ivas_format, /* i : IVAS format */ ISM_METADATA_HANDLE *hIsmMetaData, /* i : ISM metadata handle */ +#ifdef FIX_356_ISM_METADATA_SYNC + const int16_t Opt_delay_comp, /* i : Delay compensation flag */ +#endif const int16_t Opt_Headrotation, /* i : Head rotation flag */ const IVAS_QUATERNION *Quaternions, /* i : Head tracking data per subframe */ const IVAS_VECTOR3 *Pos, /* i : Listener position data per subframe */ @@ -242,6 +245,9 @@ ivas_error ivas_td_binaural_renderer_ext( const IVAS_REND_HeadRotData *headRotData, /* i : Input head positions */ const IVAS_REND_AudioObjectPosition *currentPos, /* i : Object position */ const REVERB_HANDLE hReverb, /* i : Reverberator handle */ +#ifdef FIX_356_ISM_METADATA_SYNC + const int16_t Opt_delay_comp, /* i : Delay compensation flag */ +#endif const int16_t output_frame, /* i : output frame length */ float output[][L_FRAME48k] /* i/o: SCE channels / Binaural synthesis */ ); diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index 7744a6582b..da2c7c2023 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -191,6 +191,9 @@ struct IVAS_REND int8_t rendererConfigEnabled; RENDER_CONFIG_DATA *hRendererConfig; /* Renderer config pointer */ +#ifdef FIX_356_ISM_METADATA_SYNC + int8_t delayCompensationEnabled; +#endif }; @@ -4304,6 +4307,9 @@ static ivas_error rotateFrameSba( static ivas_error renderIsmToBinaural( const input_ism *ismInput, +#ifdef FIX_356_ISM_METADATA_SYNC + const int8_t delayCompensationEnabled, +#endif IVAS_REND_AudioBuffer outAudio ) { float tmpTDRendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; @@ -4320,6 +4326,9 @@ static ivas_error renderIsmToBinaural( ismInput->base.ctx.pHeadRotData, &ismInput->currentPos, ismInput->hReverb, +#ifdef FIX_356_ISM_METADATA_SYNC + (int16_t) delayCompensationEnabled, +#endif outAudio.config.numSamplesPerChannel, tmpTDRendBuffer ) ) != IVAS_ERR_OK ) { @@ -4335,6 +4344,9 @@ static ivas_error renderIsmToBinaural( static ivas_error renderIsmToBinauralRoom( input_ism *ismInput, +#ifdef FIX_356_ISM_METADATA_SYNC + const int8_t delayCompensationEnabled, +#endif IVAS_REND_AudioBuffer outAudio ) { int16_t i; @@ -4367,6 +4379,9 @@ static ivas_error renderIsmToBinauralRoom( ismInput->base.ctx.pHeadRotData, &ismInput->currentPos, ismInput->hReverb, +#ifdef FIX_356_ISM_METADATA_SYNC + (int16_t) delayCompensationEnabled, +#endif outAudio.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) { @@ -4575,6 +4590,9 @@ static ivas_error renderIsmToSba( static ivas_error renderInputIsm( input_ism *ismInput, const IVAS_REND_AudioConfig outConfig, +#ifdef FIX_356_ISM_METADATA_SYNC + const int8_t delayCompensationEnabled, +#endif const IVAS_REND_AudioBuffer outAudio ) { ivas_error error; @@ -4604,10 +4622,18 @@ static ivas_error renderInputIsm( switch ( outConfig ) { case IVAS_REND_AUDIO_CONFIG_BINAURAL: +#ifdef FIX_356_ISM_METADATA_SYNC + error = renderIsmToBinaural( ismInput, delayCompensationEnabled, outAudio ); +#else error = renderIsmToBinaural( ismInput, outAudio ); +#endif break; case IVAS_REND_AUDIO_CONFIG_BINAURAL_ROOM: +#ifdef FIX_356_ISM_METADATA_SYNC + error = renderIsmToBinauralRoom( ismInput, delayCompensationEnabled, outAudio ); +#else error = renderIsmToBinauralRoom( ismInput, outAudio ); +#endif break; default: return IVAS_ERR_INVALID_OUTPUT_FORMAT; @@ -4641,7 +4667,11 @@ static ivas_error renderActiveInputsIsm( continue; } +#ifdef FIX_356_ISM_METADATA_SYNC + if ( ( error = renderInputIsm( pCurrentInput, hIvasRend->outputConfig, hIvasRend->delayCompensationEnabled, outAudio ) ) != IVAS_ERR_OK ) +#else if ( ( error = renderInputIsm( pCurrentInput, hIvasRend->outputConfig, outAudio ) ) != IVAS_ERR_OK ) +#endif { return error; } @@ -4720,7 +4750,11 @@ static ivas_error renderMcToBinaural( copyBufferTo2dArray( mcInput->base.inputBuffer, tmpRendBuffer ); if ( ( error = ivas_td_binaural_renderer_ext( &mcInput->tdRendWrapper, mcInput->base.inConfig, &mcInput->customLsInput, mcInput->base.ctx.pHeadRotData, NULL, +#ifdef FIX_356_ISM_METADATA_SYNC + mcInput->hReverb, 1, mcInput->base.inputBuffer.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) +#else mcInput->hReverb, mcInput->base.inputBuffer.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) +#endif { return error; } @@ -4790,7 +4824,11 @@ static ivas_error renderMcToBinauralRoom( copyBufferTo2dArray( mcInput->base.inputBuffer, tmpRendBuffer ); if ( ( error = ivas_td_binaural_renderer_ext( &mcInput->tdRendWrapper, mcInput->base.inConfig, &mcInput->customLsInput, mcInput->base.ctx.pHeadRotData, +#ifdef FIX_356_ISM_METADATA_SYNC + NULL, mcInput->hReverb, 1, mcInput->base.inputBuffer.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) +#else NULL, mcInput->hReverb, mcInput->base.inputBuffer.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) +#endif { return error; } -- GitLab From 00ba13396fedcde76cf8cb5958381d3023f71335 Mon Sep 17 00:00:00 2001 From: Sumeyra Kanik Date: Fri, 12 May 2023 21:23:26 +0200 Subject: [PATCH 02/18] Delay compensation disabled for ISM in encoder, added to decoder --- apps/decoder.c | 2 +- lib_com/delay_comp.c | 12 + lib_com/ivas_stat_com.h | 15 +- lib_dec/ivas_ism_metadata_dec.c | 17 +- lib_dec/ivas_objectRenderer_internal.c | 2 +- lib_enc/ivas_ism_metadata_enc.c | 1 - lib_rend/ivas_objectRenderer.c | 650 +++++++++++++------------ lib_rend/ivas_objectRenderer_sources.c | 1 - lib_rend/ivas_prot_rend.h | 4 + lib_util/audio_file_writer.c | 1 - 10 files changed, 379 insertions(+), 326 deletions(-) diff --git a/apps/decoder.c b/apps/decoder.c index 34ae45d396..e1a5c2a66b 100644 --- a/apps/decoder.c +++ b/apps/decoder.c @@ -1583,7 +1583,6 @@ static ivas_error decodeG192( } } - /* Write current frame */ if ( decodedGoodFrame ) { @@ -1669,6 +1668,7 @@ static ivas_error decodeG192( *------------------------------------------------------------------------------------------*/ memset( pcmBuf, 0, delayNumSamples_orig[0] * nOutChannels * sizeof( int16_t ) ); + if ( ( error = AudioFileWriter_write( afWriter, pcmBuf, delayNumSamples_orig[0] * nOutChannels ) ) != IVAS_ERR_OK ) { fprintf( stderr, "\nError writing output file: %s\n", ivas_error_to_string( error ) ); diff --git a/lib_com/delay_comp.c b/lib_com/delay_comp.c index 5557a88606..29af982512 100644 --- a/lib_com/delay_comp.c +++ b/lib_com/delay_comp.c @@ -73,6 +73,12 @@ int32_t get_delay( { delay = 0; /* All delay is compensated in the decoder with MASA */ } +#ifdef FIX_356_ISM_METADATA_SYNC + if (ivas_format == ISM_FORMAT) + { + delay = 0; /* All delay is compensated in the decoder with ISM */ + } +#endif } if ( ivas_format == SBA_FORMAT ) @@ -109,6 +115,12 @@ int32_t get_delay( { delay += IVAS_ENC_DELAY_NS; /* Compensate also the encoder delay in the decoder with MASA */ } +#ifdef FIX_356_ISM_METADATA_SYNC + if (ivas_format == ISM_FORMAT) + { + delay += IVAS_ENC_DELAY_NS; /* Compensate also the encoder delay in the decoder with ISM */ + } +#endif } } diff --git a/lib_com/ivas_stat_com.h b/lib_com/ivas_stat_com.h index 305a18951a..20073e6dd9 100644 --- a/lib_com/ivas_stat_com.h +++ b/lib_com/ivas_stat_com.h @@ -57,10 +57,10 @@ typedef struct int16_t last_ism_metadata_flag; /* last frame ism_metadata_flag */ float azimuth; /* azimuth value read from the input metadata file */ - float elevation; /* azimuth value read from the input metadata file */ - float radius; - float yaw; /* azimuth orientation value read from the input metadata file */ - float pitch; /* elevation orientation value read from the input metadata file */ + float elevation; /* elevation value read from the input metadata file */ + float radius; /* radius value read from the input metadata file */ + float yaw; /* yaw value read from the input metadata file */ + float pitch; /* pitch value read from the input metadata file */ ISM_METADATA_ANGLE position_angle; /* Angle structs for azimuth and elevation */ ISM_METADATA_ANGLE orientation_angle; /* Angle structs for yaw and pitch */ int16_t last_radius_idx; /* last frame index of coded radius */ @@ -78,6 +78,13 @@ typedef struct #ifdef FIX_435_ISM_MERGE_BUG float last_true_radius; /* last true Q radius value */ #endif +#ifdef FIX_356_ISM_METADATA_SYNC + float prev_azimuth; + float prev_elevation; + float prev_radius; + float prev_yaw; + float prev_pitch; +#endif } ISM_METADATA_FRAME, *ISM_METADATA_HANDLE; diff --git a/lib_dec/ivas_ism_metadata_dec.c b/lib_dec/ivas_ism_metadata_dec.c index 7a61424a1f..a2a79e29bb 100644 --- a/lib_dec/ivas_ism_metadata_dec.c +++ b/lib_dec/ivas_ism_metadata_dec.c @@ -244,6 +244,14 @@ ivas_error ivas_ism_metadata_dec( /* Read ISM present flags (one per object) */ for ( ch = 0; ch < *nchan_transport; ch++ ) { +#ifdef FIX_356_ISM_METADATA_SYNC + hIsmMeta[ch]->prev_azimuth = hIsmMeta[ch]->azimuth; + hIsmMeta[ch]->prev_elevation = hIsmMeta[ch]->elevation; + hIsmMeta[ch]->prev_radius = hIsmMeta[ch]->radius; + hIsmMeta[ch]->prev_yaw = hIsmMeta[ch]->yaw; + hIsmMeta[ch]->prev_pitch = hIsmMeta[ch]->pitch; + +#endif ism_imp[ch] = get_next_indice( st0, ISM_METADATA_FLAG_BITS ); if ( ism_imp[ch] > ISM_NO_META ) @@ -548,7 +556,14 @@ ivas_error ivas_ism_metadata_dec_create( st_ivas->hIsmMetaData[ch]->last_true_azimuth = 0; st_ivas->hIsmMetaData[ch]->last_true_elevation = 0; - +#ifdef FIX_356_ISM_METADATA_SYNC + // st_ivas->hIsmMetaData[ch]->last_true_radius = 1.0f; + st_ivas->hIsmMetaData[ch]->prev_azimuth = 0.0f; + st_ivas->hIsmMetaData[ch]->prev_elevation = 0.0f; + st_ivas->hIsmMetaData[ch]->prev_radius = 1.0f; + st_ivas->hIsmMetaData[ch]->prev_yaw = 0.0f; + st_ivas->hIsmMetaData[ch]->prev_pitch = 0.0f; +#endif ivas_ism_reset_metadata( st_ivas->hIsmMetaData[ch] ); } diff --git a/lib_dec/ivas_objectRenderer_internal.c b/lib_dec/ivas_objectRenderer_internal.c index 1dd3a1b1c8..9ce51cffba 100644 --- a/lib_dec/ivas_objectRenderer_internal.c +++ b/lib_dec/ivas_objectRenderer_internal.c @@ -76,7 +76,7 @@ ivas_error ivas_td_binaural_renderer( st_ivas->transport_config, st_ivas->hBinRendererTd, st_ivas->nchan_transport, LFE_CHANNEL, st_ivas->ivas_format, #ifdef FIX_356_ISM_METADATA_SYNC - st_ivas->hIsmMetaData, st_ivas->hDecoderConfig->Opt_delay_comp, st_ivas->hDecoderConfig->Opt_Headrotation, ( st_ivas->hHeadTrackData != NULL ) ? st_ivas->hHeadTrackData->Quaternions : NULL, + st_ivas->hIsmMetaData, st_ivas->hDecoderConfig->Opt_delay_comp, st_ivas->hRenderConfig->directivity, st_ivas->hDecoderConfig->Opt_Headrotation, ( st_ivas->hHeadTrackData != NULL ) ? st_ivas->hHeadTrackData->Quaternions : NULL, #else st_ivas->hIsmMetaData, st_ivas->hDecoderConfig->Opt_Headrotation, ( st_ivas->hHeadTrackData != NULL ) ? st_ivas->hHeadTrackData->Quaternions : NULL, #endif diff --git a/lib_enc/ivas_ism_metadata_enc.c b/lib_enc/ivas_ism_metadata_enc.c index a4678d6bbb..98da7f95ac 100644 --- a/lib_enc/ivas_ism_metadata_enc.c +++ b/lib_enc/ivas_ism_metadata_enc.c @@ -633,7 +633,6 @@ ivas_error ivas_ism_metadata_enc_create( st_ivas->hIsmMetaData[ch]->last_true_radius = 1.0f; #endif } - if ( ( error = ivas_ism_config( st_ivas->hEncoderConfig->ivas_total_brate, nchan_transport, n_ISms, NULL, NULL, NULL, element_brate_tmp, NULL, NULL ) ) != IVAS_ERR_OK ) { return error; diff --git a/lib_rend/ivas_objectRenderer.c b/lib_rend/ivas_objectRenderer.c index a5d3d30863..4e837f3818 100644 --- a/lib_rend/ivas_objectRenderer.c +++ b/lib_rend/ivas_objectRenderer.c @@ -258,6 +258,7 @@ ivas_error ivas_td_binaural_renderer_unwrap( ISM_METADATA_HANDLE *hIsmMetaData, /* i : ISM metadata handle */ #ifdef FIX_356_ISM_METADATA_SYNC const int16_t Opt_delay_comp, /* i : Delay compensation flag */ + const float *directivity, /* i : Directivity pattern (used for ISM) */ #endif const int16_t Opt_Headrotation, /* i : Head rotation flag */ const IVAS_QUATERNION *Quaternions, /* i : Head tracking data per subframe */ @@ -284,400 +285,417 @@ ivas_error ivas_td_binaural_renderer_unwrap( #endif subframe_length = output_frame / MAX_PARAM_SPATIAL_SUBFRAMES; -#ifndef FIX_356_ISM_METADATA_SYNC +#ifdef FIX_356_ISM_METADATA_SYNC + TDREND_Update_object_positions( hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, Opt_delay_comp, output ); +#else /* Update object position(s) */ - TDREND_Update_object_positions( hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, output ); + TDREND_Update_object_positions( hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, output ); #endif - for ( subframe_idx = 0; subframe_idx < MAX_PARAM_SPATIAL_SUBFRAMES; subframe_idx++ ) - { + for ( subframe_idx = 0; subframe_idx < MAX_PARAM_SPATIAL_SUBFRAMES; subframe_idx++ ) + { #ifdef FIX_356_ISM_METADATA_SYNC - if ( subframe_idx == subframe_update ) - { - /* Update object position(s) */ - TDREND_Update_object_positions( hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, output ); - } + if ( !Opt_delay_comp && subframe_idx == subframe_update ) + { + /* Update object position(s) */ + TDREND_Update_object_positions( hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, !Opt_delay_comp, output ); + } #endif + /* Update the listener's location/orientation */ + TDREND_Update_listener_orientation( hBinRendererTd, Opt_Headrotation, ( Quaternions != NULL ) ? &Quaternions[subframe_idx] : NULL, ( Pos != NULL ) ? &Pos[subframe_idx] : NULL ); - /* Update the listener's location/orientation */ - TDREND_Update_listener_orientation( hBinRendererTd, Opt_Headrotation, ( Quaternions != NULL ) ? &Quaternions[subframe_idx] : NULL, ( Pos != NULL ) ? &Pos[subframe_idx] : NULL ); - - if ( hReverb != NULL && hReverb->pConfig.roomAcoustics.late_reverb_on ) - { - if ( ( error = ivas_reverb_process( hReverb, transport_config, 0, output, reverb_signal, subframe_idx ) ) != IVAS_ERR_OK ) - { - return error; - } - } - - /* Render subframe */ - if ( ( error = TDREND_GetMix( hBinRendererTd, output, subframe_length, subframe_idx ) ) != IVAS_ERR_OK ) + if ( hReverb != NULL && hReverb->pConfig.roomAcoustics.late_reverb_on ) + { + if ( ( error = ivas_reverb_process( hReverb, transport_config, 0, output, reverb_signal, subframe_idx ) ) != IVAS_ERR_OK ) { return error; } } - - if ( hReverb != NULL && hReverb->pConfig.roomAcoustics.late_reverb_on ) + /* Render subframe */ + if ( ( error = TDREND_GetMix( hBinRendererTd, output, subframe_length, subframe_idx ) ) != IVAS_ERR_OK ) { - /* add reverb to rendered signals */ - v_add( reverb_signal[0], output[0], output[0], output_frame ); - v_add( reverb_signal[1], output[1], output[1], output_frame ); + return error; } + } + - return IVAS_ERR_OK; + if ( hReverb != NULL && hReverb->pConfig.roomAcoustics.late_reverb_on ) + { + /* add reverb to rendered signals */ + v_add( reverb_signal[0], output[0], output[0], output_frame ); + v_add( reverb_signal[1], output[1], output[1], output_frame ); } + return IVAS_ERR_OK; +} + - /*---------------------------------------------------------------------* - * TDREND_GetMix() - * - * Render one 5 ms subframe from the mixer - *---------------------------------------------------------------------*/ +/*---------------------------------------------------------------------* + * TDREND_GetMix() + * + * Render one 5 ms subframe from the mixer + *---------------------------------------------------------------------*/ - ivas_error TDREND_GetMix( - BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD renderer handle */ - float output[][L_FRAME48k], /* i/o: ISM object synth / rendered output in 0,1 */ - const int16_t subframe_length, /* i/o: subframe length */ - const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ - ) +ivas_error TDREND_GetMix( + BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD renderer handle */ + float output[][L_FRAME48k], /* i/o: ISM object synth / rendered output in 0,1 */ + const int16_t subframe_length, /* i/o: subframe length */ + const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ +) +{ + int16_t i; + TDREND_SRC_t *Src_p; + TDREND_SRC_SPATIAL_t *SrcSpatial_p; + TDREND_SRC_REND_t *SrcRend_p; + ivas_error error; + float output_buf[2][L_SPATIAL_SUBFR_48k]; /* Temp buffer for left/right rendered signal */ + float hrf_left_delta[SFX_SPAT_BIN_MAX_FILTER_LENGTH]; + float hrf_right_delta[SFX_SPAT_BIN_MAX_FILTER_LENGTH]; + int16_t intp_count; + error = IVAS_ERR_OK; + + /* Clear the output buffer to accumulate rendered sources */ + set_f( output_buf[0], 0.0f, subframe_length ); + set_f( output_buf[1], 0.0f, subframe_length ); + + /* Clear interpolation buffers and counter */ + set_f( hrf_left_delta, 0.0f, SFX_SPAT_BIN_MAX_FILTER_LENGTH ); + set_f( hrf_right_delta, 0.0f, SFX_SPAT_BIN_MAX_FILTER_LENGTH ); + intp_count = 0; + + /* Create the mix */ + /* Loop through the source list and render each source */ + for ( i = 0; i < hBinRendererTd->NumOfSrcs; i++ ) { - int16_t i; - TDREND_SRC_t *Src_p; - TDREND_SRC_SPATIAL_t *SrcSpatial_p; - TDREND_SRC_REND_t *SrcRend_p; - ivas_error error; - float output_buf[2][L_SPATIAL_SUBFR_48k]; /* Temp buffer for left/right rendered signal */ - float hrf_left_delta[SFX_SPAT_BIN_MAX_FILTER_LENGTH]; - float hrf_right_delta[SFX_SPAT_BIN_MAX_FILTER_LENGTH]; - int16_t intp_count; - error = IVAS_ERR_OK; - - /* Clear the output buffer to accumulate rendered sources */ - set_f( output_buf[0], 0.0f, subframe_length ); - set_f( output_buf[1], 0.0f, subframe_length ); - - /* Clear interpolation buffers and counter */ - set_f( hrf_left_delta, 0.0f, SFX_SPAT_BIN_MAX_FILTER_LENGTH ); - set_f( hrf_right_delta, 0.0f, SFX_SPAT_BIN_MAX_FILTER_LENGTH ); - intp_count = 0; - - /* Create the mix */ - /* Loop through the source list and render each source */ - for ( i = 0; i < hBinRendererTd->NumOfSrcs; i++ ) - { - Src_p = hBinRendererTd->Sources[i]; - SrcSpatial_p = Src_p->SrcSpatial_p; - SrcRend_p = Src_p->SrcRend_p; + Src_p = hBinRendererTd->Sources[i]; + SrcSpatial_p = Src_p->SrcSpatial_p; + SrcRend_p = Src_p->SrcRend_p; - /* Update rendering params if needed */ - if ( hBinRendererTd->Listener_p->PoseUpdated || SrcSpatial_p->Updated ) - { - TDREND_SRC_REND_UpdateFiltersFromSpatialParams( hBinRendererTd, SrcRend_p, SrcSpatial_p, Src_p->hrf_left_prev, - Src_p->hrf_right_prev, hrf_left_delta, hrf_right_delta, &intp_count, &Src_p->filterlength, &Src_p->itd, &Src_p->Gain, Src_p, subframe_idx ); - } + /* Update rendering params if needed */ + if ( hBinRendererTd->Listener_p->PoseUpdated || SrcSpatial_p->Updated ) + { + TDREND_SRC_REND_UpdateFiltersFromSpatialParams( hBinRendererTd, SrcRend_p, SrcSpatial_p, Src_p->hrf_left_prev, + Src_p->hrf_right_prev, hrf_left_delta, hrf_right_delta, &intp_count, &Src_p->filterlength, &Src_p->itd, &Src_p->Gain, Src_p, subframe_idx ); + } - /* Render source if needed */ - if ( ( SrcRend_p->InputAvailable == TRUE ) && ( SrcRend_p->PlayStatus == TDREND_PLAYSTATUS_PLAYING ) ) - { - error = TDREND_REND_RenderSourceHRFilt( Src_p, hrf_left_delta, hrf_right_delta, intp_count, output_buf, subframe_length ); - } + /* Render source if needed */ + if ( ( SrcRend_p->InputAvailable == TRUE ) && ( SrcRend_p->PlayStatus == TDREND_PLAYSTATUS_PLAYING ) ) + { + error = TDREND_REND_RenderSourceHRFilt( Src_p, hrf_left_delta, hrf_right_delta, intp_count, output_buf, subframe_length ); } + } - /* Populate output variable */ - mvr2r( output_buf[0], output[0] + subframe_idx * subframe_length, subframe_length ); /* Left */ - mvr2r( output_buf[1], output[1] + subframe_idx * subframe_length, subframe_length ); /* Right */ + /* Populate output variable */ + mvr2r( output_buf[0], output[0] + subframe_idx * subframe_length, subframe_length ); /* Left */ + mvr2r( output_buf[1], output[1] + subframe_idx * subframe_length, subframe_length ); /* Right */ - /* Clear the PoseUpdated and Source position update flags */ - TDREND_Clear_Update_flags( hBinRendererTd ); + /* Clear the PoseUpdated and Source position update flags */ + TDREND_Clear_Update_flags( hBinRendererTd ); - return error; - } + return error; +} - /*---------------------------------------------------------------------* - * TDREND_Clear_Update_flags() - * - * Initializes the audio mixer module - *---------------------------------------------------------------------*/ +/*---------------------------------------------------------------------* + * TDREND_Clear_Update_flags() + * + * Initializes the audio mixer module + *---------------------------------------------------------------------*/ - static void TDREND_Clear_Update_flags( - BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd /* i/o: TD renderer handle */ - ) +static void TDREND_Clear_Update_flags( + BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd /* i/o: TD renderer handle */ +) +{ + int16_t i; + + hBinRendererTd->Listener_p->PoseUpdated = FALSE; + + for ( i = 0; i < hBinRendererTd->NumOfSrcs; i++ ) { - int16_t i; + hBinRendererTd->Sources[i]->SrcSpatial_p->Updated = FALSE; + } + + return; +} - hBinRendererTd->Listener_p->PoseUpdated = FALSE; - for ( i = 0; i < hBinRendererTd->NumOfSrcs; i++ ) - { - hBinRendererTd->Sources[i]->SrcSpatial_p->Updated = FALSE; - } +/*---------------------------------------------------------------------* + * TDREND_Update_object_positions() + * + * Update object position(s) + *---------------------------------------------------------------------*/ - return; - } +void TDREND_Update_object_positions( + BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o : TD Renderer handle */ + const int16_t num_src, /* i : number of sources to render */ + const int16_t lfe_idx, /* i : Input LFE index */ + const IVAS_FORMAT in_format, /* i : Format of input sources */ + const ISM_METADATA_HANDLE *hIsmMetaData, /* i : Input metadata for ISM objects */ +#ifdef FIX_356_ISM_METADATA_SYNC + const int16_t Opt_delay_comp, +#endif + float output[][L_FRAME48k] /* i/o: SCE/MC channels */ +) +{ + TDREND_DirAtten_t *DirAtten_p; + int16_t nS; + float Pos[3]; + float Dir[3]; + int16_t c_indx; + DirAtten_p = hBinRendererTd->DirAtten_p; - /*---------------------------------------------------------------------* - * TDREND_Update_object_positions() - * - * Update object position(s) - *---------------------------------------------------------------------*/ - - void TDREND_Update_object_positions( - BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o : TD Renderer handle */ - const int16_t num_src, /* i : number of sources to render */ - const int16_t lfe_idx, /* i : Input LFE index */ - const IVAS_FORMAT in_format, /* i : Format of input sources */ - const ISM_METADATA_HANDLE *hIsmMetaData, /* i : Input metadata for ISM objects */ - float output[][L_FRAME48k] /* i/o: SCE/MC channels */ - ) + /* For each source, write the frame data to the source object*/ + c_indx = 0; + for ( nS = 0; nS < num_src; nS++ ) { - TDREND_DirAtten_t *DirAtten_p; - int16_t nS; - float Pos[3]; - float Dir[3]; - int16_t c_indx; - - DirAtten_p = hBinRendererTd->DirAtten_p; + if ( !( in_format == MC_FORMAT && nS == lfe_idx ) ) /* Skip LFE for MC */ + { + hBinRendererTd->Sources[c_indx]->InputFrame_p = output[nS]; + hBinRendererTd->Sources[c_indx]->SrcRend_p->InputAvailable = TRUE; + c_indx++; + } - /* For each source, write the frame data to the source object*/ - c_indx = 0; - for ( nS = 0; nS < num_src; nS++ ) + if ( in_format == ISM_FORMAT ) { - if ( !( in_format == MC_FORMAT && nS == lfe_idx ) ) /* Skip LFE for MC */ + /* Update the source positions */ + /* Source position and direction */ +#ifdef FIX_356_ISM_METADATA_SYNC + if ( !Opt_delay_comp ) { - hBinRendererTd->Sources[c_indx]->InputFrame_p = output[nS]; - hBinRendererTd->Sources[c_indx]->SrcRend_p->InputAvailable = TRUE; - c_indx++; + angles_to_vec( hIsmMetaData[nS]->prev_radius, hIsmMetaData[nS]->prev_azimuth, hIsmMetaData[nS]->prev_elevation, Pos ); + angles_to_vec( 1.0f, hIsmMetaData[nS]->prev_yaw, hIsmMetaData[nS]->prev_pitch, Dir ); } - - if ( in_format == ISM_FORMAT ) + else { - - /* Update the source positions */ - /* Source position and direction */ angles_to_vec( hIsmMetaData[nS]->radius, hIsmMetaData[nS]->azimuth, hIsmMetaData[nS]->elevation, Pos ); angles_to_vec( 1.0f, hIsmMetaData[nS]->yaw, hIsmMetaData[nS]->pitch, Dir ); - /* Source directivity info */ - DirAtten_p->ConeInnerAngle = 360.0f; - DirAtten_p->ConeOuterAngle = 360.0f; - DirAtten_p->ConeOuterGain = 1.0f; + } +#else + angles_to_vec( hIsmMetaData[nS]->radius, hIsmMetaData[nS]->azimuth, hIsmMetaData[nS]->elevation, Pos ); + angles_to_vec( 1.0f, hIsmMetaData[nS]->yaw, hIsmMetaData[nS]->pitch, Dir ); +#endif + /* Source directivity info */ + DirAtten_p->ConeInnerAngle = 360.0f; + DirAtten_p->ConeOuterAngle = 360.0f; + DirAtten_p->ConeOuterGain = 1.0f; - TDREND_MIX_SRC_SetPos( hBinRendererTd, nS, Pos ); - TDREND_MIX_SRC_SetDirAtten( hBinRendererTd, nS, DirAtten_p ); - TDREND_MIX_SRC_SetPlayState( hBinRendererTd, nS, TDREND_PLAYSTATUS_PLAYING ); + TDREND_MIX_SRC_SetPos( hBinRendererTd, nS, Pos ); + TDREND_MIX_SRC_SetDirAtten( hBinRendererTd, nS, DirAtten_p ); + TDREND_MIX_SRC_SetPlayState( hBinRendererTd, nS, TDREND_PLAYSTATUS_PLAYING ); - TDREND_MIX_SRC_SetDir( hBinRendererTd, nS, Dir ); - } - } + TDREND_MIX_SRC_SetDir( hBinRendererTd, nS, Dir ); - return; + } } + return; +} - /*---------------------------------------------------------------------* - * TDREND_Update_listener_orientation() - * - * Update listener orientation (s) - *---------------------------------------------------------------------*/ - - void TDREND_Update_listener_orientation( - BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD Renderer handle */ - const int16_t headRotEnabled, /* i : Headrotation flag */ - const IVAS_QUATERNION *headPosition, /* i : Listener orientation */ - const IVAS_VECTOR3 *Pos /* i : Listener Position */ - ) - { - float FrontVec[3]; - float UpVec[3]; - float Rmat[3][3]; - float Pos_p[3]; - if ( headRotEnabled ) - { - /* Obtain head rotation matrix */ - QuatToRotMat( *headPosition, Rmat ); - /* Apply rotation matrix to looking vector [1;0;0] */ - FrontVec[0] = Rmat[0][0]; - FrontVec[1] = Rmat[0][1]; - FrontVec[2] = Rmat[0][2]; - /* Apply rotation matrix to up vector [0;0;1] */ - UpVec[0] = Rmat[2][0]; - UpVec[1] = Rmat[2][1]; - UpVec[2] = Rmat[2][2]; - /* Input position */ - Pos_p[0] = ( *Pos ).x; - Pos_p[1] = ( *Pos ).y; - Pos_p[2] = ( *Pos ).z; - } - else - { - /* Oriented with looking vector along the x axis */ - FrontVec[0] = 1.0f; - FrontVec[1] = 0.0f; - FrontVec[2] = 0.0f; - /* Oriented with up vector along the z axis */ - UpVec[0] = 0.0f; - UpVec[1] = 0.0f; - UpVec[2] = 1.0f; - /* Listener at the origin */ - Pos_p[0] = 0.0f; - Pos_p[1] = 0.0f; - Pos_p[2] = 0.0f; - } +/*---------------------------------------------------------------------* + * TDREND_Update_listener_orientation() + * + * Update listener orientation (s) + *---------------------------------------------------------------------*/ - /* Set the listener position and orientation:*/ - TDREND_MIX_LIST_SetPos( hBinRendererTd, Pos_p ); - TDREND_MIX_LIST_SetOrient( hBinRendererTd, FrontVec, UpVec ); +void TDREND_Update_listener_orientation( + BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD Renderer handle */ + const int16_t headRotEnabled, /* i : Headrotation flag */ + const IVAS_QUATERNION *headPosition, /* i : Listener orientation */ + const IVAS_VECTOR3 *Pos /* i : Listener Position */ +) +{ + float FrontVec[3]; + float UpVec[3]; + float Rmat[3][3]; + float Pos_p[3]; - return; + if ( headRotEnabled ) + { + /* Obtain head rotation matrix */ + QuatToRotMat( *headPosition, Rmat ); + /* Apply rotation matrix to looking vector [1;0;0] */ + FrontVec[0] = Rmat[0][0]; + FrontVec[1] = Rmat[0][1]; + FrontVec[2] = Rmat[0][2]; + /* Apply rotation matrix to up vector [0;0;1] */ + UpVec[0] = Rmat[2][0]; + UpVec[1] = Rmat[2][1]; + UpVec[2] = Rmat[2][2]; + /* Input position */ + Pos_p[0] = ( *Pos ).x; + Pos_p[1] = ( *Pos ).y; + Pos_p[2] = ( *Pos ).z; + } + else + { + /* Oriented with looking vector along the x axis */ + FrontVec[0] = 1.0f; + FrontVec[1] = 0.0f; + FrontVec[2] = 0.0f; + /* Oriented with up vector along the z axis */ + UpVec[0] = 0.0f; + UpVec[1] = 0.0f; + UpVec[2] = 1.0f; + /* Listener at the origin */ + Pos_p[0] = 0.0f; + Pos_p[1] = 0.0f; + Pos_p[2] = 0.0f; } + /* Set the listener position and orientation:*/ + TDREND_MIX_LIST_SetPos( hBinRendererTd, Pos_p ); + TDREND_MIX_LIST_SetOrient( hBinRendererTd, FrontVec, UpVec ); - /*---------------------------------------------------------------------* - * ivas_td_binaural_open_ext() - * - * - *---------------------------------------------------------------------*/ + return; +} - ivas_error ivas_td_binaural_open_ext( - TDREND_WRAPPER * pTDRend, - IVAS_REND_AudioConfig inConfig, - RENDER_CONFIG_DATA * hRendCfg, /* i : Renderer configuration */ - LSSETUP_CUSTOM_STRUCT * customLsInput, - const int32_t outFs ) - { - int16_t nchan_transport; - AUDIO_CONFIG transport_config; - IVAS_FORMAT ivas_format; - IVAS_OUTPUT_SETUP hTransSetup; - ivas_error error; - float *directivity = NULL; - if ( inConfig != IVAS_REND_AUDIO_CONFIG_LS_CUSTOM ) - { - if ( ( error = getAudioConfigNumChannels( inConfig, &nchan_transport ) ) != IVAS_ERR_OK ) - { - return error; - } - } - else - { - nchan_transport = customLsInput->num_spk; - } +/*---------------------------------------------------------------------* + * ivas_td_binaural_open_ext() + * + * + *---------------------------------------------------------------------*/ - transport_config = getIvasAudioConfigFromRendAudioConfig( inConfig ); - ivas_format = ( getAudioConfigType( inConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) ? MC_FORMAT : ISM_FORMAT; - hTransSetup.ls_azimuth = customLsInput->ls_azimuth; - hTransSetup.ls_elevation = customLsInput->ls_elevation; +ivas_error ivas_td_binaural_open_ext( + TDREND_WRAPPER *pTDRend, + IVAS_REND_AudioConfig inConfig, + RENDER_CONFIG_DATA *hRendCfg, /* i : Renderer configuration */ + LSSETUP_CUSTOM_STRUCT *customLsInput, + const int32_t outFs ) +{ + int16_t nchan_transport; + AUDIO_CONFIG transport_config; + IVAS_FORMAT ivas_format; + IVAS_OUTPUT_SETUP hTransSetup; + ivas_error error; + float *directivity = NULL; - if ( NULL != hRendCfg ) + if ( inConfig != IVAS_REND_AUDIO_CONFIG_LS_CUSTOM ) + { + if ( ( error = getAudioConfigNumChannels( inConfig, &nchan_transport ) ) != IVAS_ERR_OK ) { - directivity = hRendCfg->directivity; + return error; } + } + else + { + nchan_transport = customLsInput->num_spk; + } + + transport_config = getIvasAudioConfigFromRendAudioConfig( inConfig ); + ivas_format = ( getAudioConfigType( inConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) ? MC_FORMAT : ISM_FORMAT; + hTransSetup.ls_azimuth = customLsInput->ls_azimuth; + hTransSetup.ls_elevation = customLsInput->ls_elevation; - return ivas_td_binaural_open_unwrap( &pTDRend->hHrtfTD, outFs, nchan_transport, ivas_format, transport_config, directivity, hTransSetup, &pTDRend->hBinRendererTd, &pTDRend->binaural_latency_ns ); + if ( NULL != hRendCfg ) + { + directivity = hRendCfg->directivity; } + return ivas_td_binaural_open_unwrap( &pTDRend->hHrtfTD, outFs, nchan_transport, ivas_format, transport_config, directivity, hTransSetup, &pTDRend->hBinRendererTd, &pTDRend->binaural_latency_ns ); +} + - /*---------------------------------------------------------------------* - * ivas_td_binaural_renderer_ext() - * - * Receives the current frames for the object streams, updates metadata - * and renders the current frame. - *---------------------------------------------------------------------*/ - - ivas_error ivas_td_binaural_renderer_ext( - const TDREND_WRAPPER *pTDRend, /* i : TD Renderer wrapper structure */ - const IVAS_REND_AudioConfig inConfig, /* i : Input audio configuration */ - const LSSETUP_CUSTOM_STRUCT *customLsInput, /* i : Input custom loudspeaker layout */ - const IVAS_REND_HeadRotData *headRotData, /* i : Input head positions */ - const IVAS_REND_AudioObjectPosition *currentPos, /* i : Object position */ - const REVERB_HANDLE hReverb, /* i : Reverberator handle */ +/*---------------------------------------------------------------------* + * ivas_td_binaural_renderer_ext() + * + * Receives the current frames for the object streams, updates metadata + * and renders the current frame. + *---------------------------------------------------------------------*/ + +ivas_error ivas_td_binaural_renderer_ext( + const TDREND_WRAPPER *pTDRend, /* i : TD Renderer wrapper structure */ + const IVAS_REND_AudioConfig inConfig, /* i : Input audio configuration */ + const LSSETUP_CUSTOM_STRUCT *customLsInput, /* i : Input custom loudspeaker layout */ + const IVAS_REND_HeadRotData *headRotData, /* i : Input head positions */ + const IVAS_REND_AudioObjectPosition *currentPos, /* i : Object position */ + const REVERB_HANDLE hReverb, /* i : Reverberator handle */ #ifdef FIX_356_ISM_METADATA_SYNC - const int16_t Opt_delay_comp, /* i : Delay compensation flag */ + const int16_t Opt_delay_comp, /* i : Delay compensation flag */ #endif - const int16_t output_frame, /* i : output frame length */ - float output[][L_FRAME48k] /* i/o: SCE channels / Binaural synthesis */ - ) + const int16_t output_frame, /* i : output frame length */ + float output[][L_FRAME48k] /* i/o: SCE channels / Binaural synthesis */ +) +{ + ISM_METADATA_FRAME hIsmMetaDataFrame; + ISM_METADATA_HANDLE hIsmMetaData[1]; + int16_t lfe_idx; + int16_t num_src; + IVAS_FORMAT ivas_format; + IVAS_REND_AudioConfigType inConfigType; + AUDIO_CONFIG transport_config; + ivas_error error; + + push_wmops( "ivas_td_binaural_renderer_ext" ); + + inConfigType = getAudioConfigType( inConfig ); + lfe_idx = LFE_CHANNEL; + hIsmMetaData[0] = NULL; + + if ( inConfigType == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) { - ISM_METADATA_FRAME hIsmMetaDataFrame; - ISM_METADATA_HANDLE hIsmMetaData[1]; - int16_t lfe_idx; - int16_t num_src; - IVAS_FORMAT ivas_format; - IVAS_REND_AudioConfigType inConfigType; - AUDIO_CONFIG transport_config; - ivas_error error; - - push_wmops( "ivas_td_binaural_renderer_ext" ); - - inConfigType = getAudioConfigType( inConfig ); - lfe_idx = LFE_CHANNEL; - hIsmMetaData[0] = NULL; - - if ( inConfigType == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) + ivas_format = MC_FORMAT; + transport_config = getIvasAudioConfigFromRendAudioConfig( inConfig ); + if ( inConfig != IVAS_REND_AUDIO_CONFIG_LS_CUSTOM ) { - ivas_format = MC_FORMAT; - transport_config = getIvasAudioConfigFromRendAudioConfig( inConfig ); - if ( inConfig != IVAS_REND_AUDIO_CONFIG_LS_CUSTOM ) + if ( ( error = getAudioConfigNumChannels( inConfig, &num_src ) ) != IVAS_ERR_OK ) { - if ( ( error = getAudioConfigNumChannels( inConfig, &num_src ) ) != IVAS_ERR_OK ) - { - return error; - } - } - else - { - lfe_idx = ( customLsInput->num_lfe > 0 ) ? customLsInput->lfe_idx[0] : -1; - num_src = customLsInput->num_spk + customLsInput->num_lfe; + return error; } } else { - ivas_format = ISM_FORMAT; - num_src = 1; - transport_config = AUDIO_CONFIG_ISM1; - hIsmMetaData[0] = &hIsmMetaDataFrame; - hIsmMetaData[0]->azimuth = currentPos->azimuth; - hIsmMetaData[0]->elevation = currentPos->elevation; - hIsmMetaData[0]->yaw = currentPos->yaw; - hIsmMetaData[0]->pitch = currentPos->pitch; - hIsmMetaData[0]->radius = currentPos->radius; + lfe_idx = ( customLsInput->num_lfe > 0 ) ? customLsInput->lfe_idx[0] : -1; + num_src = customLsInput->num_spk + customLsInput->num_lfe; } + } + else + { + ivas_format = ISM_FORMAT; + num_src = 1; + transport_config = AUDIO_CONFIG_ISM1; + hIsmMetaData[0] = &hIsmMetaDataFrame; + hIsmMetaData[0]->azimuth = currentPos->azimuth; + hIsmMetaData[0]->elevation = currentPos->elevation; + hIsmMetaData[0]->yaw = currentPos->yaw; + hIsmMetaData[0]->pitch = currentPos->pitch; + hIsmMetaData[0]->radius = currentPos->radius; + } #ifdef FIX_356_ISM_METADATA_SYNC - if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, Opt_delay_comp, headRotData->headRotEnabled, + if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, Opt_delay_comp, NULL, headRotData->headRotEnabled, #else - if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, + if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, #endif - ( headRotData != NULL ) ? headRotData->headPositions : NULL, - ( headRotData != NULL ) ? headRotData->Pos : NULL, output, output_frame ) ) != IVAS_ERR_OK ) - { - return error; - } + ( headRotData != NULL ) ? headRotData->headPositions : NULL, + ( headRotData != NULL ) ? headRotData->Pos : NULL, output, output_frame ) ) != IVAS_ERR_OK ) + { + return error; + } - pop_wmops(); + pop_wmops(); - return IVAS_ERR_OK; - } + return IVAS_ERR_OK; +} - /*---------------------------------------------------------------------* - * angles_to_vec() - * - * Convert azimuth and elevation angles to position/orientation vector - *---------------------------------------------------------------------*/ - - static void angles_to_vec( - const float radius, /* i : radius */ - const float azimuth, /* i : Azimuth angle */ - const float elevation, /* i : Elevation angle */ - float *vec /* o : Pos/Dir vector */ - ) - { - vec[0] = radius * cosf( elevation * PI_OVER_180 ) * cosf( azimuth * PI_OVER_180 ); - vec[1] = radius * cosf( elevation * PI_OVER_180 ) * sinf( azimuth * PI_OVER_180 ); - vec[2] = radius * sinf( elevation * PI_OVER_180 ); +/*---------------------------------------------------------------------* + * angles_to_vec() + * + * Convert azimuth and elevation angles to position/orientation vector + *---------------------------------------------------------------------*/ - return; - } +static void angles_to_vec( + const float radius, /* i : radius */ + const float azimuth, /* i : Azimuth angle */ + const float elevation, /* i : Elevation angle */ + float *vec /* o : Pos/Dir vector */ +) +{ + vec[0] = radius * cosf( elevation * PI_OVER_180 ) * cosf( azimuth * PI_OVER_180 ); + vec[1] = radius * cosf( elevation * PI_OVER_180 ) * sinf( azimuth * PI_OVER_180 ); + vec[2] = radius * sinf( elevation * PI_OVER_180 ); + + return; +} diff --git a/lib_rend/ivas_objectRenderer_sources.c b/lib_rend/ivas_objectRenderer_sources.c index 91253748b2..4b58104c17 100644 --- a/lib_rend/ivas_objectRenderer_sources.c +++ b/lib_rend/ivas_objectRenderer_sources.c @@ -219,7 +219,6 @@ static void TDREND_SRC_REND_Init( /* Internal state */ SrcRend_p->InputAvailable = FALSE; SrcRend_p->PlayStatus = TDREND_PLAYSTATUS_INITIAL; - /* SrcGain */ for ( nC = 0; nC < SPAT_BIN_MAX_INPUT_CHANNELS; nC++ ) { diff --git a/lib_rend/ivas_prot_rend.h b/lib_rend/ivas_prot_rend.h index b3232656ae..81839e17ac 100644 --- a/lib_rend/ivas_prot_rend.h +++ b/lib_rend/ivas_prot_rend.h @@ -230,6 +230,7 @@ ivas_error ivas_td_binaural_renderer_unwrap( ISM_METADATA_HANDLE *hIsmMetaData, /* i : ISM metadata handle */ #ifdef FIX_356_ISM_METADATA_SYNC const int16_t Opt_delay_comp, /* i : Delay compensation flag */ + const float *directivity, /* i : Directivity pattern (used for ISM) */ #endif const int16_t Opt_Headrotation, /* i : Head rotation flag */ const IVAS_QUATERNION *Quaternions, /* i : Head tracking data per subframe */ @@ -297,6 +298,9 @@ void TDREND_Update_object_positions( const int16_t lfe_idx, /* i : Input LFE index */ const IVAS_FORMAT in_format, /* i : Format of input sources */ const ISM_METADATA_HANDLE *hIsmMetaData, /* i : Input metadata for ISM objects */ +#ifdef FIX_356_ISM_METADATA_SYNC + const int16_t Opt_delay_comp, +#endif float output[][L_FRAME48k] /* i/o: SCE/MC channels */ ); diff --git a/lib_util/audio_file_writer.c b/lib_util/audio_file_writer.c index 81084dbd95..0ed64d02c5 100644 --- a/lib_util/audio_file_writer.c +++ b/lib_util/audio_file_writer.c @@ -151,7 +151,6 @@ ivas_error AudioFileWriter_write( uint32_t numSamples ) { ivas_error error = IVAS_ERR_OK; - if ( self->rawFile ) { if ( fwrite( samples, sizeof( int16_t ), numSamples, self->rawFile ) != numSamples ) -- GitLab From 0e6b46c06f4989fdd83e06a3e1b7735857f8e92a Mon Sep 17 00:00:00 2001 From: Sumeyra Kanik Date: Mon, 15 May 2023 11:47:02 +0200 Subject: [PATCH 03/18] Update object position at subframe level --- lib_com/ivas_ism_com.c | 8 ++++++ lib_dec/ivas_dec.c | 3 +++ lib_dec/ivas_ism_metadata_dec.c | 9 +------ lib_rend/ivas_objectRenderer.c | 36 ++++++++++++++++++++++---- lib_rend/ivas_objectRenderer_sources.c | 7 +++++ lib_rend/ivas_prot_rend.h | 6 +++++ 6 files changed, 56 insertions(+), 13 deletions(-) diff --git a/lib_com/ivas_ism_com.c b/lib_com/ivas_ism_com.c index 38a34a15bd..d2d0d7ec3a 100644 --- a/lib_com/ivas_ism_com.c +++ b/lib_com/ivas_ism_com.c @@ -337,6 +337,14 @@ void ivas_ism_reset_metadata( hIsmMeta->yaw = 0.0f; hIsmMeta->pitch = 0.0f; hIsmMeta->radius = 1.0f; +#ifdef FIX_356_ISM_METADATA_SYNC + // st_ivas->hIsmMetaData[ch]->last_true_radius = 1.0f; + hIsmMeta->prev_azimuth = 0.0f; + hIsmMeta->prev_elevation = 0.0f; + hIsmMeta->prev_radius = 1.0f; + hIsmMeta->prev_yaw = 0.0f; + hIsmMeta->prev_pitch = 0.0f; +#endif return; } diff --git a/lib_dec/ivas_dec.c b/lib_dec/ivas_dec.c index 11cc33af48..43f3c7e0ef 100644 --- a/lib_dec/ivas_dec.c +++ b/lib_dec/ivas_dec.c @@ -227,6 +227,9 @@ ivas_error ivas_dec( /* Binaural rendering */ if ( st_ivas->renderer_type == RENDERER_BINAURAL_OBJECTS_TD ) { +#ifdef FIX_356_ISM_METADATA_SYNC + dbgwrite( &output[0], sizeof( float ), L_FRAME48k, 1, "out.float" ); +#endif if ( ( ivas_td_binaural_renderer( st_ivas, output, output_frame ) ) != IVAS_ERR_OK ) { return error; diff --git a/lib_dec/ivas_ism_metadata_dec.c b/lib_dec/ivas_ism_metadata_dec.c index a2a79e29bb..1ffba4b282 100644 --- a/lib_dec/ivas_ism_metadata_dec.c +++ b/lib_dec/ivas_ism_metadata_dec.c @@ -556,14 +556,7 @@ ivas_error ivas_ism_metadata_dec_create( st_ivas->hIsmMetaData[ch]->last_true_azimuth = 0; st_ivas->hIsmMetaData[ch]->last_true_elevation = 0; -#ifdef FIX_356_ISM_METADATA_SYNC - // st_ivas->hIsmMetaData[ch]->last_true_radius = 1.0f; - st_ivas->hIsmMetaData[ch]->prev_azimuth = 0.0f; - st_ivas->hIsmMetaData[ch]->prev_elevation = 0.0f; - st_ivas->hIsmMetaData[ch]->prev_radius = 1.0f; - st_ivas->hIsmMetaData[ch]->prev_yaw = 0.0f; - st_ivas->hIsmMetaData[ch]->prev_pitch = 0.0f; -#endif + ivas_ism_reset_metadata( st_ivas->hIsmMetaData[ch] ); } diff --git a/lib_rend/ivas_objectRenderer.c b/lib_rend/ivas_objectRenderer.c index 4e837f3818..ccfd61fa7e 100644 --- a/lib_rend/ivas_objectRenderer.c +++ b/lib_rend/ivas_objectRenderer.c @@ -286,6 +286,17 @@ ivas_error ivas_td_binaural_renderer_unwrap( subframe_length = output_frame / MAX_PARAM_SPATIAL_SUBFRAMES; #ifdef FIX_356_ISM_METADATA_SYNC + int16_t c_indx, nS; + c_indx = 0; + for (nS = 0; nS < num_src; nS++) + { + if ( !( ivas_format == MC_FORMAT && nS == lfe_idx ) ) /* Skip LFE for MC */ + { + hBinRendererTd->Sources[c_indx]->InputFrame_p = output[nS]; + hBinRendererTd->Sources[c_indx]->SrcRend_p->InputAvailable = TRUE; + c_indx++; + } + } TDREND_Update_object_positions( hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, Opt_delay_comp, output ); #else /* Update object position(s) */ @@ -312,7 +323,11 @@ ivas_error ivas_td_binaural_renderer_unwrap( } /* Render subframe */ +#ifdef FIX_356_ISM_METADATA_SYNC + if ( ( error = TDREND_GetMix( hBinRendererTd, output, subframe_update, subframe_length, subframe_idx ) ) != IVAS_ERR_OK ) +#else if ( ( error = TDREND_GetMix( hBinRendererTd, output, subframe_length, subframe_idx ) ) != IVAS_ERR_OK ) +#endif { return error; } @@ -339,8 +354,11 @@ ivas_error ivas_td_binaural_renderer_unwrap( ivas_error TDREND_GetMix( BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD renderer handle */ float output[][L_FRAME48k], /* i/o: ISM object synth / rendered output in 0,1 */ - const int16_t subframe_length, /* i/o: subframe length */ - const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ +#ifdef FIX_356_ISM_METADATA_SYNC + const int16_t subframe_update, +#endif + const int16_t subframe_length, /* i/o: subframe length */ + const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ ) { int16_t i; @@ -375,7 +393,13 @@ ivas_error TDREND_GetMix( if ( hBinRendererTd->Listener_p->PoseUpdated || SrcSpatial_p->Updated ) { TDREND_SRC_REND_UpdateFiltersFromSpatialParams( hBinRendererTd, SrcRend_p, SrcSpatial_p, Src_p->hrf_left_prev, - Src_p->hrf_right_prev, hrf_left_delta, hrf_right_delta, &intp_count, &Src_p->filterlength, &Src_p->itd, &Src_p->Gain, Src_p, subframe_idx ); + Src_p->hrf_right_prev, hrf_left_delta, hrf_right_delta, &intp_count, &Src_p->filterlength, &Src_p->itd, &Src_p->Gain, Src_p, +#ifdef FIX_356_ISM_METADATA_SYNC + subframe_update, +#endif + subframe_idx + + ); } /* Render source if needed */ @@ -446,16 +470,19 @@ void TDREND_Update_object_positions( DirAtten_p = hBinRendererTd->DirAtten_p; /* For each source, write the frame data to the source object*/ +#ifndef FIX_356_ISM_METADATA_SYNC c_indx = 0; +#endif for ( nS = 0; nS < num_src; nS++ ) { +#ifndef FIX_356_ISM_METADATA_SYNC if ( !( in_format == MC_FORMAT && nS == lfe_idx ) ) /* Skip LFE for MC */ { hBinRendererTd->Sources[c_indx]->InputFrame_p = output[nS]; hBinRendererTd->Sources[c_indx]->SrcRend_p->InputAvailable = TRUE; c_indx++; } - +#endif if ( in_format == ISM_FORMAT ) { /* Update the source positions */ @@ -485,7 +512,6 @@ void TDREND_Update_object_positions( TDREND_MIX_SRC_SetPlayState( hBinRendererTd, nS, TDREND_PLAYSTATUS_PLAYING ); TDREND_MIX_SRC_SetDir( hBinRendererTd, nS, Dir ); - } } diff --git a/lib_rend/ivas_objectRenderer_sources.c b/lib_rend/ivas_objectRenderer_sources.c index 4b58104c17..27c6b55e2d 100644 --- a/lib_rend/ivas_objectRenderer_sources.c +++ b/lib_rend/ivas_objectRenderer_sources.c @@ -259,6 +259,9 @@ void TDREND_SRC_REND_UpdateFiltersFromSpatialParams( int16_t *itd, /* o: ITD value */ float *Gain, /* o: Gain value */ TDREND_SRC_t *Src_p, /* i/o: Source pointer */ +#ifdef FIX_356_ISM_METADATA_SYNC + const int16_t subframe_update, +#endif const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ ) { @@ -353,7 +356,11 @@ void TDREND_SRC_REND_UpdateFiltersFromSpatialParams( azim_delta = ( azim_delta > 180.0f ) ? ( azim_delta - 360 ) : ( ( azim_delta < -180.0f ) ? ( azim_delta + 360 ) : ( azim_delta ) ); /* map to -180:180 range */ *intp_count = min( MAX_INTERPOLATION_STEPS, max( (int16_t) ( fabsf( azim_delta ) * MAX_ANGULAR_STEP_INV ), (int16_t) ( fabsf( elev_delta ) * MAX_ANGULAR_STEP_INV ) ) ); +#ifdef FIX_356_ISM_METADATA_SYNC + if ( ( *intp_count > 0 ) && subframe_idx == subframe_update ) +#else if ( ( *intp_count > 0 ) && subframe_idx == 0 ) +#endif { /* Set deltas for interpolation */ v_sub( hrf_left, hrf_left_prev, hrf_left_delta, *filterlength ); diff --git a/lib_rend/ivas_prot_rend.h b/lib_rend/ivas_prot_rend.h index 81839e17ac..689a96e866 100644 --- a/lib_rend/ivas_prot_rend.h +++ b/lib_rend/ivas_prot_rend.h @@ -281,6 +281,9 @@ void ivas_td_binaural_close( ivas_error TDREND_GetMix( BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD renderer handle */ float output[][L_FRAME48k], /* i/o: ISM object synth / rendered output in 0,1 */ +#ifdef FIX_356_ISM_METADATA_SYNC + const int16_t Opt_delay_comp, +#endif const int16_t subframe_length, /* i/o: subframe length */ const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ ); @@ -372,6 +375,9 @@ void TDREND_SRC_REND_UpdateFiltersFromSpatialParams( int16_t *itd, /* o: ITD value */ float *Gain, /* o: Gain value */ TDREND_SRC_t *Src_p, +#ifdef FIX_356_ISM_METADATA_SYNC + const int16_t subframe_update, +#endif const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ ); -- GitLab From a9413a894be5119f013d325ca3ee99b3d871e9b4 Mon Sep 17 00:00:00 2001 From: Sumeyra Kanik Date: Mon, 15 May 2023 14:25:55 +0200 Subject: [PATCH 04/18] Code cleanup, always update subframe 2 --- apps/decoder.c | 4 --- lib_com/ivas_ism_com.c | 8 ----- lib_com/ivas_stat_com.h | 7 ---- lib_dec/ivas_dec.c | 3 -- lib_dec/ivas_ism_metadata_dec.c | 8 ----- lib_dec/ivas_objectRenderer_internal.c | 4 --- lib_dec/ivas_stat_dec.h | 3 -- lib_dec/lib_dec.c | 11 ------ lib_dec/lib_dec.h | 6 +--- lib_rend/ivas_objectRenderer.c | 50 +++++--------------------- lib_rend/ivas_prot_rend.h | 13 ------- lib_rend/lib_rend.c | 38 -------------------- 12 files changed, 10 insertions(+), 145 deletions(-) diff --git a/apps/decoder.c b/apps/decoder.c index e1a5c2a66b..5a37e9bd4a 100644 --- a/apps/decoder.c +++ b/apps/decoder.c @@ -354,11 +354,7 @@ int main( /*------------------------------------------------------------------------------------------* * Configure the decoder *------------------------------------------------------------------------------------------*/ -#ifdef FIX_356_ISM_METADATA_SYNC - if ( ( error = IVAS_DEC_Configure( hIvasDec, arg.output_Fs, arg.outputFormat, arg.customLsOutputEnabled, arg.hrtfReaderEnabled, arg.enableHeadRotation, arg.renderConfigEnabled, arg.delayCompensationEnabled ) ) != IVAS_ERR_OK ) -#else if ( ( error = IVAS_DEC_Configure( hIvasDec, arg.output_Fs, arg.outputFormat, arg.customLsOutputEnabled, arg.hrtfReaderEnabled, arg.enableHeadRotation, arg.renderConfigEnabled ) ) != IVAS_ERR_OK ) -#endif { fprintf( stderr, "\nConfigure failed: %s\n\n", IVAS_DEC_GetErrorMessage( error ) ); goto cleanup; diff --git a/lib_com/ivas_ism_com.c b/lib_com/ivas_ism_com.c index d2d0d7ec3a..38a34a15bd 100644 --- a/lib_com/ivas_ism_com.c +++ b/lib_com/ivas_ism_com.c @@ -337,14 +337,6 @@ void ivas_ism_reset_metadata( hIsmMeta->yaw = 0.0f; hIsmMeta->pitch = 0.0f; hIsmMeta->radius = 1.0f; -#ifdef FIX_356_ISM_METADATA_SYNC - // st_ivas->hIsmMetaData[ch]->last_true_radius = 1.0f; - hIsmMeta->prev_azimuth = 0.0f; - hIsmMeta->prev_elevation = 0.0f; - hIsmMeta->prev_radius = 1.0f; - hIsmMeta->prev_yaw = 0.0f; - hIsmMeta->prev_pitch = 0.0f; -#endif return; } diff --git a/lib_com/ivas_stat_com.h b/lib_com/ivas_stat_com.h index 20073e6dd9..f1bf01cc73 100644 --- a/lib_com/ivas_stat_com.h +++ b/lib_com/ivas_stat_com.h @@ -78,13 +78,6 @@ typedef struct #ifdef FIX_435_ISM_MERGE_BUG float last_true_radius; /* last true Q radius value */ #endif -#ifdef FIX_356_ISM_METADATA_SYNC - float prev_azimuth; - float prev_elevation; - float prev_radius; - float prev_yaw; - float prev_pitch; -#endif } ISM_METADATA_FRAME, *ISM_METADATA_HANDLE; diff --git a/lib_dec/ivas_dec.c b/lib_dec/ivas_dec.c index 43f3c7e0ef..11cc33af48 100644 --- a/lib_dec/ivas_dec.c +++ b/lib_dec/ivas_dec.c @@ -227,9 +227,6 @@ ivas_error ivas_dec( /* Binaural rendering */ if ( st_ivas->renderer_type == RENDERER_BINAURAL_OBJECTS_TD ) { -#ifdef FIX_356_ISM_METADATA_SYNC - dbgwrite( &output[0], sizeof( float ), L_FRAME48k, 1, "out.float" ); -#endif if ( ( ivas_td_binaural_renderer( st_ivas, output, output_frame ) ) != IVAS_ERR_OK ) { return error; diff --git a/lib_dec/ivas_ism_metadata_dec.c b/lib_dec/ivas_ism_metadata_dec.c index 1ffba4b282..7a61424a1f 100644 --- a/lib_dec/ivas_ism_metadata_dec.c +++ b/lib_dec/ivas_ism_metadata_dec.c @@ -244,14 +244,6 @@ ivas_error ivas_ism_metadata_dec( /* Read ISM present flags (one per object) */ for ( ch = 0; ch < *nchan_transport; ch++ ) { -#ifdef FIX_356_ISM_METADATA_SYNC - hIsmMeta[ch]->prev_azimuth = hIsmMeta[ch]->azimuth; - hIsmMeta[ch]->prev_elevation = hIsmMeta[ch]->elevation; - hIsmMeta[ch]->prev_radius = hIsmMeta[ch]->radius; - hIsmMeta[ch]->prev_yaw = hIsmMeta[ch]->yaw; - hIsmMeta[ch]->prev_pitch = hIsmMeta[ch]->pitch; - -#endif ism_imp[ch] = get_next_indice( st0, ISM_METADATA_FLAG_BITS ); if ( ism_imp[ch] > ISM_NO_META ) diff --git a/lib_dec/ivas_objectRenderer_internal.c b/lib_dec/ivas_objectRenderer_internal.c index 9ce51cffba..f1c1fc04f2 100644 --- a/lib_dec/ivas_objectRenderer_internal.c +++ b/lib_dec/ivas_objectRenderer_internal.c @@ -75,10 +75,6 @@ ivas_error ivas_td_binaural_renderer( st_ivas->hReverb, st_ivas->transport_config, st_ivas->hBinRendererTd, st_ivas->nchan_transport, LFE_CHANNEL, st_ivas->ivas_format, -#ifdef FIX_356_ISM_METADATA_SYNC - st_ivas->hIsmMetaData, st_ivas->hDecoderConfig->Opt_delay_comp, st_ivas->hRenderConfig->directivity, st_ivas->hDecoderConfig->Opt_Headrotation, ( st_ivas->hHeadTrackData != NULL ) ? st_ivas->hHeadTrackData->Quaternions : NULL, -#else st_ivas->hIsmMetaData, st_ivas->hDecoderConfig->Opt_Headrotation, ( st_ivas->hHeadTrackData != NULL ) ? st_ivas->hHeadTrackData->Quaternions : NULL, -#endif ( st_ivas->hHeadTrackData != NULL ) ? st_ivas->hHeadTrackData->Pos : NULL, output, output_frame ); } diff --git a/lib_dec/ivas_stat_dec.h b/lib_dec/ivas_stat_dec.h index 39e13e771a..ad453cd5bd 100644 --- a/lib_dec/ivas_stat_dec.h +++ b/lib_dec/ivas_stat_dec.h @@ -1145,9 +1145,6 @@ typedef struct decoder_config_structure int16_t orientation_tracking; /* indicates orientation tracking type */ float no_diegetic_pan; int16_t Opt_AMR_WB; /* flag indicating AMR-WB IO mode */ -#ifdef FIX_356_ISM_METADATA_SYNC - int16_t Opt_delay_comp; /* flag indicating delay compensation active */ -#endif /* temp. development parameters */ #ifdef DEBUGGING diff --git a/lib_dec/lib_dec.c b/lib_dec/lib_dec.c index 5791fda0c6..99529372eb 100644 --- a/lib_dec/lib_dec.c +++ b/lib_dec/lib_dec.c @@ -226,9 +226,6 @@ static void init_decoder_config( hDecoderConfig->Opt_RendConfigCustom = 0; hDecoderConfig->orientation_tracking = orientation_tracking; hDecoderConfig->no_diegetic_pan = no_diegetic_pan; -#ifdef FIX_356_ISM_METADATA_SYNC - hDecoderConfig->Opt_delay_comp = 0; -#endif return; } @@ -391,12 +388,7 @@ ivas_error IVAS_DEC_Configure( const int16_t customLsOutputEnabled, /* i : enable custom loudspeaker setup handle */ const int16_t hrtfReaderEnabled, /* i : enable HRTF binary file input */ const int16_t enableHeadRotation, /* i : enable head rotation for binaural output */ -#ifdef FIX_356_ISM_METADATA_SYNC - const int16_t renderConfigEnabled, /* i : enable Renderer config. file for binaural output */ - const int16_t delayCompensationEnabled /* i : enable delay compensation */ -#else const int16_t renderConfigEnabled /* i : enable Renderer config. file for binaural output */ -#endif ) { Decoder_Struct *st_ivas; @@ -446,9 +438,6 @@ ivas_error IVAS_DEC_Configure( hDecoderConfig->Opt_Headrotation = enableHeadRotation; hDecoderConfig->Opt_HRTF_binary = hrtfReaderEnabled; hDecoderConfig->Opt_RendConfigCustom = renderConfigEnabled; -#ifdef FIX_356_ISM_METADATA_SYNC - hDecoderConfig->Opt_delay_comp = delayCompensationEnabled; -#endif /* Set decoder parameters to initial values */ if ( ( error = ivas_init_decoder_front( st_ivas ) ) != IVAS_ERR_OK ) diff --git a/lib_dec/lib_dec.h b/lib_dec/lib_dec.h index 16aa8a66dd..3950f44e03 100644 --- a/lib_dec/lib_dec.h +++ b/lib_dec/lib_dec.h @@ -130,12 +130,8 @@ ivas_error IVAS_DEC_Configure( const int16_t customLsOutputEnabled, /* i : enable custom loudspeaker setup handle */ const int16_t hrtfReaderEnabled, /* i : enable HRTF binary file input */ const int16_t enableHeadRotation, /* i : enable head rotation for binaural output */ -#ifdef FIX_356_ISM_METADATA_SYNC - const int16_t renderConfigEnabled, /* i : enable Renderer config. file for binaural output */ - const int16_t delayCompensationEnabled /* i : enable delay compensation */ -#else const int16_t renderConfigEnabled /* i : enable Renderer config. file for binaural output */ -#endif + ); void IVAS_DEC_Close( diff --git a/lib_rend/ivas_objectRenderer.c b/lib_rend/ivas_objectRenderer.c index ccfd61fa7e..91dc7c5d2e 100644 --- a/lib_rend/ivas_objectRenderer.c +++ b/lib_rend/ivas_objectRenderer.c @@ -256,10 +256,6 @@ ivas_error ivas_td_binaural_renderer_unwrap( const int16_t lfe_idx, /* i : LFE channel index */ const IVAS_FORMAT ivas_format, /* i : IVAS format */ ISM_METADATA_HANDLE *hIsmMetaData, /* i : ISM metadata handle */ -#ifdef FIX_356_ISM_METADATA_SYNC - const int16_t Opt_delay_comp, /* i : Delay compensation flag */ - const float *directivity, /* i : Directivity pattern (used for ISM) */ -#endif const int16_t Opt_Headrotation, /* i : Head rotation flag */ const IVAS_QUATERNION *Quaternions, /* i : Head tracking data per subframe */ const IVAS_VECTOR3 *Pos, /* i : Listener position data per subframe */ @@ -271,23 +267,16 @@ ivas_error ivas_td_binaural_renderer_unwrap( int16_t subframe_idx; float reverb_signal[BINAURAL_CHANNELS][L_FRAME48k]; ivas_error error; -#ifdef FIX_356_ISM_METADATA_SYNC - int16_t subframe_update; - if ( Opt_delay_comp ) - { - subframe_update = 0; - } - else - { - subframe_update = 2; /* To be verified */ - } -#endif subframe_length = output_frame / MAX_PARAM_SPATIAL_SUBFRAMES; #ifdef FIX_356_ISM_METADATA_SYNC + int16_t subframe_update; int16_t c_indx, nS; + + subframe_update = 2; c_indx = 0; + for (nS = 0; nS < num_src; nS++) { if ( !( ivas_format == MC_FORMAT && nS == lfe_idx ) ) /* Skip LFE for MC */ @@ -297,18 +286,18 @@ ivas_error ivas_td_binaural_renderer_unwrap( c_indx++; } } - TDREND_Update_object_positions( hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, Opt_delay_comp, output ); #else /* Update object position(s) */ TDREND_Update_object_positions( hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, output ); #endif + for ( subframe_idx = 0; subframe_idx < MAX_PARAM_SPATIAL_SUBFRAMES; subframe_idx++ ) { #ifdef FIX_356_ISM_METADATA_SYNC - if ( !Opt_delay_comp && subframe_idx == subframe_update ) + if ( subframe_idx == subframe_update ) { /* Update object position(s) */ - TDREND_Update_object_positions( hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, !Opt_delay_comp, output ); + TDREND_Update_object_positions( hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, output ); } #endif /* Update the listener's location/orientation */ @@ -455,9 +444,6 @@ void TDREND_Update_object_positions( const int16_t lfe_idx, /* i : Input LFE index */ const IVAS_FORMAT in_format, /* i : Format of input sources */ const ISM_METADATA_HANDLE *hIsmMetaData, /* i : Input metadata for ISM objects */ -#ifdef FIX_356_ISM_METADATA_SYNC - const int16_t Opt_delay_comp, -#endif float output[][L_FRAME48k] /* i/o: SCE/MC channels */ ) { @@ -487,21 +473,9 @@ void TDREND_Update_object_positions( { /* Update the source positions */ /* Source position and direction */ -#ifdef FIX_356_ISM_METADATA_SYNC - if ( !Opt_delay_comp ) - { - angles_to_vec( hIsmMetaData[nS]->prev_radius, hIsmMetaData[nS]->prev_azimuth, hIsmMetaData[nS]->prev_elevation, Pos ); - angles_to_vec( 1.0f, hIsmMetaData[nS]->prev_yaw, hIsmMetaData[nS]->prev_pitch, Dir ); - } - else - { - angles_to_vec( hIsmMetaData[nS]->radius, hIsmMetaData[nS]->azimuth, hIsmMetaData[nS]->elevation, Pos ); - angles_to_vec( 1.0f, hIsmMetaData[nS]->yaw, hIsmMetaData[nS]->pitch, Dir ); - } -#else angles_to_vec( hIsmMetaData[nS]->radius, hIsmMetaData[nS]->azimuth, hIsmMetaData[nS]->elevation, Pos ); angles_to_vec( 1.0f, hIsmMetaData[nS]->yaw, hIsmMetaData[nS]->pitch, Dir ); -#endif + /* Source directivity info */ DirAtten_p->ConeInnerAngle = 360.0f; DirAtten_p->ConeOuterAngle = 360.0f; @@ -638,9 +612,6 @@ ivas_error ivas_td_binaural_renderer_ext( const IVAS_REND_HeadRotData *headRotData, /* i : Input head positions */ const IVAS_REND_AudioObjectPosition *currentPos, /* i : Object position */ const REVERB_HANDLE hReverb, /* i : Reverberator handle */ -#ifdef FIX_356_ISM_METADATA_SYNC - const int16_t Opt_delay_comp, /* i : Delay compensation flag */ -#endif const int16_t output_frame, /* i : output frame length */ float output[][L_FRAME48k] /* i/o: SCE channels / Binaural synthesis */ ) @@ -690,11 +661,8 @@ ivas_error ivas_td_binaural_renderer_ext( hIsmMetaData[0]->radius = currentPos->radius; } -#ifdef FIX_356_ISM_METADATA_SYNC - if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, Opt_delay_comp, NULL, headRotData->headRotEnabled, -#else + if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, -#endif ( headRotData != NULL ) ? headRotData->headPositions : NULL, ( headRotData != NULL ) ? headRotData->Pos : NULL, output, output_frame ) ) != IVAS_ERR_OK ) { diff --git a/lib_rend/ivas_prot_rend.h b/lib_rend/ivas_prot_rend.h index 689a96e866..a9697688a8 100644 --- a/lib_rend/ivas_prot_rend.h +++ b/lib_rend/ivas_prot_rend.h @@ -228,10 +228,6 @@ ivas_error ivas_td_binaural_renderer_unwrap( const int16_t lfe_idx, /* i : LFE channel index */ const IVAS_FORMAT ivas_format, /* i : IVAS format */ ISM_METADATA_HANDLE *hIsmMetaData, /* i : ISM metadata handle */ -#ifdef FIX_356_ISM_METADATA_SYNC - const int16_t Opt_delay_comp, /* i : Delay compensation flag */ - const float *directivity, /* i : Directivity pattern (used for ISM) */ -#endif const int16_t Opt_Headrotation, /* i : Head rotation flag */ const IVAS_QUATERNION *Quaternions, /* i : Head tracking data per subframe */ const IVAS_VECTOR3 *Pos, /* i : Listener position data per subframe */ @@ -246,9 +242,6 @@ ivas_error ivas_td_binaural_renderer_ext( const IVAS_REND_HeadRotData *headRotData, /* i : Input head positions */ const IVAS_REND_AudioObjectPosition *currentPos, /* i : Object position */ const REVERB_HANDLE hReverb, /* i : Reverberator handle */ -#ifdef FIX_356_ISM_METADATA_SYNC - const int16_t Opt_delay_comp, /* i : Delay compensation flag */ -#endif const int16_t output_frame, /* i : output frame length */ float output[][L_FRAME48k] /* i/o: SCE channels / Binaural synthesis */ ); @@ -281,9 +274,6 @@ void ivas_td_binaural_close( ivas_error TDREND_GetMix( BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD renderer handle */ float output[][L_FRAME48k], /* i/o: ISM object synth / rendered output in 0,1 */ -#ifdef FIX_356_ISM_METADATA_SYNC - const int16_t Opt_delay_comp, -#endif const int16_t subframe_length, /* i/o: subframe length */ const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ ); @@ -301,9 +291,6 @@ void TDREND_Update_object_positions( const int16_t lfe_idx, /* i : Input LFE index */ const IVAS_FORMAT in_format, /* i : Format of input sources */ const ISM_METADATA_HANDLE *hIsmMetaData, /* i : Input metadata for ISM objects */ -#ifdef FIX_356_ISM_METADATA_SYNC - const int16_t Opt_delay_comp, -#endif float output[][L_FRAME48k] /* i/o: SCE/MC channels */ ); diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index da2c7c2023..7744a6582b 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -191,9 +191,6 @@ struct IVAS_REND int8_t rendererConfigEnabled; RENDER_CONFIG_DATA *hRendererConfig; /* Renderer config pointer */ -#ifdef FIX_356_ISM_METADATA_SYNC - int8_t delayCompensationEnabled; -#endif }; @@ -4307,9 +4304,6 @@ static ivas_error rotateFrameSba( static ivas_error renderIsmToBinaural( const input_ism *ismInput, -#ifdef FIX_356_ISM_METADATA_SYNC - const int8_t delayCompensationEnabled, -#endif IVAS_REND_AudioBuffer outAudio ) { float tmpTDRendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; @@ -4326,9 +4320,6 @@ static ivas_error renderIsmToBinaural( ismInput->base.ctx.pHeadRotData, &ismInput->currentPos, ismInput->hReverb, -#ifdef FIX_356_ISM_METADATA_SYNC - (int16_t) delayCompensationEnabled, -#endif outAudio.config.numSamplesPerChannel, tmpTDRendBuffer ) ) != IVAS_ERR_OK ) { @@ -4344,9 +4335,6 @@ static ivas_error renderIsmToBinaural( static ivas_error renderIsmToBinauralRoom( input_ism *ismInput, -#ifdef FIX_356_ISM_METADATA_SYNC - const int8_t delayCompensationEnabled, -#endif IVAS_REND_AudioBuffer outAudio ) { int16_t i; @@ -4379,9 +4367,6 @@ static ivas_error renderIsmToBinauralRoom( ismInput->base.ctx.pHeadRotData, &ismInput->currentPos, ismInput->hReverb, -#ifdef FIX_356_ISM_METADATA_SYNC - (int16_t) delayCompensationEnabled, -#endif outAudio.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) { @@ -4590,9 +4575,6 @@ static ivas_error renderIsmToSba( static ivas_error renderInputIsm( input_ism *ismInput, const IVAS_REND_AudioConfig outConfig, -#ifdef FIX_356_ISM_METADATA_SYNC - const int8_t delayCompensationEnabled, -#endif const IVAS_REND_AudioBuffer outAudio ) { ivas_error error; @@ -4622,18 +4604,10 @@ static ivas_error renderInputIsm( switch ( outConfig ) { case IVAS_REND_AUDIO_CONFIG_BINAURAL: -#ifdef FIX_356_ISM_METADATA_SYNC - error = renderIsmToBinaural( ismInput, delayCompensationEnabled, outAudio ); -#else error = renderIsmToBinaural( ismInput, outAudio ); -#endif break; case IVAS_REND_AUDIO_CONFIG_BINAURAL_ROOM: -#ifdef FIX_356_ISM_METADATA_SYNC - error = renderIsmToBinauralRoom( ismInput, delayCompensationEnabled, outAudio ); -#else error = renderIsmToBinauralRoom( ismInput, outAudio ); -#endif break; default: return IVAS_ERR_INVALID_OUTPUT_FORMAT; @@ -4667,11 +4641,7 @@ static ivas_error renderActiveInputsIsm( continue; } -#ifdef FIX_356_ISM_METADATA_SYNC - if ( ( error = renderInputIsm( pCurrentInput, hIvasRend->outputConfig, hIvasRend->delayCompensationEnabled, outAudio ) ) != IVAS_ERR_OK ) -#else if ( ( error = renderInputIsm( pCurrentInput, hIvasRend->outputConfig, outAudio ) ) != IVAS_ERR_OK ) -#endif { return error; } @@ -4750,11 +4720,7 @@ static ivas_error renderMcToBinaural( copyBufferTo2dArray( mcInput->base.inputBuffer, tmpRendBuffer ); if ( ( error = ivas_td_binaural_renderer_ext( &mcInput->tdRendWrapper, mcInput->base.inConfig, &mcInput->customLsInput, mcInput->base.ctx.pHeadRotData, NULL, -#ifdef FIX_356_ISM_METADATA_SYNC - mcInput->hReverb, 1, mcInput->base.inputBuffer.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) -#else mcInput->hReverb, mcInput->base.inputBuffer.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) -#endif { return error; } @@ -4824,11 +4790,7 @@ static ivas_error renderMcToBinauralRoom( copyBufferTo2dArray( mcInput->base.inputBuffer, tmpRendBuffer ); if ( ( error = ivas_td_binaural_renderer_ext( &mcInput->tdRendWrapper, mcInput->base.inConfig, &mcInput->customLsInput, mcInput->base.ctx.pHeadRotData, -#ifdef FIX_356_ISM_METADATA_SYNC - NULL, mcInput->hReverb, 1, mcInput->base.inputBuffer.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) -#else NULL, mcInput->hReverb, mcInput->base.inputBuffer.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) -#endif { return error; } -- GitLab From db676f088c033bec5182b5ec9dd10973bb21ee48 Mon Sep 17 00:00:00 2001 From: Sumeyra Kanik Date: Tue, 16 May 2023 13:50:10 +0200 Subject: [PATCH 05/18] Correct the delay printout for decoder --- lib_com/ivas_cnst.h | 3 +++ lib_dec/lib_dec.c | 8 ++++++++ lib_rend/ivas_objectRenderer.c | 19 ++----------------- lib_rend/ivas_objectRenderer_sources.c | 5 +---- lib_rend/ivas_prot_rend.h | 3 --- 5 files changed, 14 insertions(+), 24 deletions(-) diff --git a/lib_com/ivas_cnst.h b/lib_com/ivas_cnst.h index 91f1127788..4a6cc1dba9 100755 --- a/lib_com/ivas_cnst.h +++ b/lib_com/ivas_cnst.h @@ -364,6 +364,9 @@ typedef enum #endif #define ISM_EXTENDED_METADATA_BITS 1 #define ISM_METADATA_RS_MAX_FRAMES 5 /* Number of frames with opposite extended metadata flags before switching */ +#ifdef FIX_356_ISM_METADATA_SYNC +#define ISM_METADATA_DELAY_SUBFRAME 2 /* Number of subframes to delay metadata to sync with audio */ +#endif /* Parametric ISM */ #define MAX_PARAM_ISM_NBANDS 11 diff --git a/lib_dec/lib_dec.c b/lib_dec/lib_dec.c index 17c718bb48..4e535f98d7 100644 --- a/lib_dec/lib_dec.c +++ b/lib_dec/lib_dec.c @@ -1594,6 +1594,14 @@ ivas_error IVAS_DEC_GetDelay( nSamples[2] = (int16_t) roundf( (float) st_ivas->binaural_latency_ns * hDecoderConfig->output_Fs / 1000000000.f ); nSamples[0] = nSamples[1] + nSamples[2]; +#ifdef FIX_356_ISM_METADATA_SYNC + if (st_ivas->ivas_format == ISM_FORMAT) + { + /* note: in ISM, all delay is compensated at the decoder by default, so subtract the encoder delay for print-out */ + nSamples[1] -= NS2SA(hDecoderConfig->output_Fs, IVAS_ENC_DELAY_NS); + } +#endif + *timeScale = hDecoderConfig->output_Fs; return IVAS_ERR_OK; diff --git a/lib_rend/ivas_objectRenderer.c b/lib_rend/ivas_objectRenderer.c index a857bf1170..3129eaca4f 100644 --- a/lib_rend/ivas_objectRenderer.c +++ b/lib_rend/ivas_objectRenderer.c @@ -290,10 +290,7 @@ ivas_error ivas_td_binaural_renderer_unwrap( subframe_length = output_frame / MAX_PARAM_SPATIAL_SUBFRAMES; #ifdef FIX_356_ISM_METADATA_SYNC - int16_t subframe_update; int16_t c_indx, nS; - - subframe_update = 2; c_indx = 0; for (nS = 0; nS < num_src; nS++) @@ -313,7 +310,7 @@ ivas_error ivas_td_binaural_renderer_unwrap( for ( subframe_idx = 0; subframe_idx < MAX_PARAM_SPATIAL_SUBFRAMES; subframe_idx++ ) { #ifdef FIX_356_ISM_METADATA_SYNC - if ( subframe_idx == subframe_update ) + if ( subframe_idx == ISM_METADATA_DELAY_SUBFRAME ) { /* Update object position(s) */ TDREND_Update_object_positions( hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, output ); @@ -335,11 +332,7 @@ ivas_error ivas_td_binaural_renderer_unwrap( } /* Render subframe */ -#ifdef FIX_356_ISM_METADATA_SYNC - if ( ( error = TDREND_GetMix( hBinRendererTd, output, subframe_update, subframe_length, subframe_idx ) ) != IVAS_ERR_OK ) -#else if ( ( error = TDREND_GetMix( hBinRendererTd, output, subframe_length, subframe_idx ) ) != IVAS_ERR_OK ) -#endif { return error; } @@ -369,9 +362,6 @@ ivas_error TDREND_GetMix( float *output[], /* i/o: ISM object synth / rendered output in 0,1 */ #else float output[][L_FRAME48k], /* i/o: ISM object synth / rendered output in 0,1 */ -#endif -#ifdef FIX_356_ISM_METADATA_SYNC - const int16_t subframe_update, #endif const int16_t subframe_length, /* i/o: subframe length */ const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ @@ -418,12 +408,7 @@ ivas_error TDREND_GetMix( { TDREND_SRC_REND_UpdateFiltersFromSpatialParams( hBinRendererTd, SrcRend_p, SrcSpatial_p, Src_p->hrf_left_prev, Src_p->hrf_right_prev, hrf_left_delta, hrf_right_delta, &intp_count, &Src_p->filterlength, &Src_p->itd, &Src_p->Gain, Src_p, -#ifdef FIX_356_ISM_METADATA_SYNC - subframe_update, -#endif - subframe_idx - - ); + subframe_idx ); } /* Render source if needed */ diff --git a/lib_rend/ivas_objectRenderer_sources.c b/lib_rend/ivas_objectRenderer_sources.c index 27c6b55e2d..f622764482 100644 --- a/lib_rend/ivas_objectRenderer_sources.c +++ b/lib_rend/ivas_objectRenderer_sources.c @@ -259,9 +259,6 @@ void TDREND_SRC_REND_UpdateFiltersFromSpatialParams( int16_t *itd, /* o: ITD value */ float *Gain, /* o: Gain value */ TDREND_SRC_t *Src_p, /* i/o: Source pointer */ -#ifdef FIX_356_ISM_METADATA_SYNC - const int16_t subframe_update, -#endif const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ ) { @@ -357,7 +354,7 @@ void TDREND_SRC_REND_UpdateFiltersFromSpatialParams( *intp_count = min( MAX_INTERPOLATION_STEPS, max( (int16_t) ( fabsf( azim_delta ) * MAX_ANGULAR_STEP_INV ), (int16_t) ( fabsf( elev_delta ) * MAX_ANGULAR_STEP_INV ) ) ); #ifdef FIX_356_ISM_METADATA_SYNC - if ( ( *intp_count > 0 ) && subframe_idx == subframe_update ) + if ( ( *intp_count > 0 ) && subframe_idx == ISM_METADATA_DELAY_SUBFRAME ) #else if ( ( *intp_count > 0 ) && subframe_idx == 0 ) #endif diff --git a/lib_rend/ivas_prot_rend.h b/lib_rend/ivas_prot_rend.h index 5e25aa0da3..b43a048f38 100644 --- a/lib_rend/ivas_prot_rend.h +++ b/lib_rend/ivas_prot_rend.h @@ -389,9 +389,6 @@ void TDREND_SRC_REND_UpdateFiltersFromSpatialParams( int16_t *itd, /* o: ITD value */ float *Gain, /* o: Gain value */ TDREND_SRC_t *Src_p, -#ifdef FIX_356_ISM_METADATA_SYNC - const int16_t subframe_update, -#endif const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ ); -- GitLab From 6aee116631b5ce6b957f2c12e06608ac59af45db Mon Sep 17 00:00:00 2001 From: vaclav Date: Tue, 16 May 2023 16:44:29 +0200 Subject: [PATCH 06/18] code reduction within FIX_356_ISM_METADATA_SYNC --- lib_com/delay_comp.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/lib_com/delay_comp.c b/lib_com/delay_comp.c index 29af982512..518a28aba7 100644 --- a/lib_com/delay_comp.c +++ b/lib_com/delay_comp.c @@ -69,14 +69,15 @@ int32_t get_delay( { delay = IVAS_ENC_DELAY_NS; - if ( ivas_format == MASA_FORMAT ) +#ifdef FIX_356_ISM_METADATA_SYNC + if ( ivas_format == ISM_FORMAT || ivas_format == MASA_FORMAT ) { - delay = 0; /* All delay is compensated in the decoder with MASA */ + delay = 0; /* All delay is compensated in the decoder with MASA/ISM */ } -#ifdef FIX_356_ISM_METADATA_SYNC - if (ivas_format == ISM_FORMAT) +#else + if ( ivas_format == MASA_FORMAT ) { - delay = 0; /* All delay is compensated in the decoder with ISM */ + delay = 0; /* All delay is compensated in the decoder with MASA */ } #endif } @@ -110,15 +111,15 @@ int32_t get_delay( delay += IVAS_FB_DEC_DELAY_NS; } - - if ( ivas_format == MASA_FORMAT ) +#ifdef FIX_356_ISM_METADATA_SYNC + if ( ivas_format == ISM_FORMAT || ivas_format == MASA_FORMAT ) { - delay += IVAS_ENC_DELAY_NS; /* Compensate also the encoder delay in the decoder with MASA */ + delay += IVAS_ENC_DELAY_NS; /* Compensate also the encoder delay in the decoder with ISM/MASA */ } -#ifdef FIX_356_ISM_METADATA_SYNC - if (ivas_format == ISM_FORMAT) +#else + if ( ivas_format == MASA_FORMAT ) { - delay += IVAS_ENC_DELAY_NS; /* Compensate also the encoder delay in the decoder with ISM */ + delay += IVAS_ENC_DELAY_NS; /* Compensate also the encoder delay in the decoder with MASA */ } #endif } -- GitLab From 0a6223abc4bf2a70715344dfa93b8ccd6b342998 Mon Sep 17 00:00:00 2001 From: vaclav Date: Tue, 16 May 2023 16:50:58 +0200 Subject: [PATCH 07/18] fix warnings + clang-formatting --- lib_com/ivas_stat_com.h | 6 +++--- lib_dec/ivas_objectRenderer_internal.c | 5 +++++ lib_dec/lib_dec.c | 6 +++--- lib_rend/ivas_objectRenderer.c | 21 ++++++++++++++------- lib_rend/ivas_prot_rend.h | 7 ++++++- 5 files changed, 31 insertions(+), 14 deletions(-) diff --git a/lib_com/ivas_stat_com.h b/lib_com/ivas_stat_com.h index 6e98a6ba3c..1414051f51 100644 --- a/lib_com/ivas_stat_com.h +++ b/lib_com/ivas_stat_com.h @@ -61,9 +61,9 @@ typedef struct #ifdef ISM_NON_DIEGETIC_PAN int16_t non_diegetic_flag; /* Non-diegetic (non-headtracked) object flag */ #endif - float azimuth; /* azimuth value read from the input metadata file */ - float elevation; /* elevation value read from the input metadata file */ - float radius; /* radius value read from the input metadata file */ + float azimuth; /* azimuth value read from the input metadata file */ + float elevation; /* elevation value read from the input metadata file */ + float radius; /* radius value read from the input metadata file */ float yaw; /* yaw value read from the input metadata file */ float pitch; /* pitch value read from the input metadata file */ ISM_METADATA_ANGLE position_angle; /* Angle structs for azimuth and elevation */ diff --git a/lib_dec/ivas_objectRenderer_internal.c b/lib_dec/ivas_objectRenderer_internal.c index 873c75938a..6d701be0a5 100644 --- a/lib_dec/ivas_objectRenderer_internal.c +++ b/lib_dec/ivas_objectRenderer_internal.c @@ -129,8 +129,13 @@ void ObjRenderIVASSubframe( for ( subframe_idx = first_sf; subframe_idx < last_sf; subframe_idx++ ) { output_frame = st_ivas->hTcBuffer->subframe_nbslots[subframe_idx] * st_ivas->hTcBuffer->n_samples_granularity; + /* Update object position(s) */ +#ifdef FIX_356_ISM_METADATA_SYNC + TDREND_Update_object_positions( st_ivas->hBinRendererTd, st_ivas->nchan_transport, st_ivas->ivas_format, st_ivas->hIsmMetaData ); +#else TDREND_Update_object_positions( st_ivas->hBinRendererTd, st_ivas->nchan_transport, LFE_CHANNEL, st_ivas->ivas_format, st_ivas->hIsmMetaData, tc_local ); +#endif /* Update the listener's location/orientation */ TDREND_Update_listener_orientation( st_ivas->hBinRendererTd, diff --git a/lib_dec/lib_dec.c b/lib_dec/lib_dec.c index 4e535f98d7..7eb660a6b3 100644 --- a/lib_dec/lib_dec.c +++ b/lib_dec/lib_dec.c @@ -1594,11 +1594,11 @@ ivas_error IVAS_DEC_GetDelay( nSamples[2] = (int16_t) roundf( (float) st_ivas->binaural_latency_ns * hDecoderConfig->output_Fs / 1000000000.f ); nSamples[0] = nSamples[1] + nSamples[2]; -#ifdef FIX_356_ISM_METADATA_SYNC - if (st_ivas->ivas_format == ISM_FORMAT) +#ifdef FIX_356_ISM_METADATA_SYNC + if ( st_ivas->ivas_format == ISM_FORMAT ) { /* note: in ISM, all delay is compensated at the decoder by default, so subtract the encoder delay for print-out */ - nSamples[1] -= NS2SA(hDecoderConfig->output_Fs, IVAS_ENC_DELAY_NS); + nSamples[1] -= NS2SA( hDecoderConfig->output_Fs, IVAS_ENC_DELAY_NS ); } #endif diff --git a/lib_rend/ivas_objectRenderer.c b/lib_rend/ivas_objectRenderer.c index 3129eaca4f..a7a8df5eed 100644 --- a/lib_rend/ivas_objectRenderer.c +++ b/lib_rend/ivas_objectRenderer.c @@ -293,7 +293,7 @@ ivas_error ivas_td_binaural_renderer_unwrap( int16_t c_indx, nS; c_indx = 0; - for (nS = 0; nS < num_src; nS++) + for ( nS = 0; nS < num_src; nS++ ) { if ( !( ivas_format == MC_FORMAT && nS == lfe_idx ) ) /* Skip LFE for MC */ { @@ -313,7 +313,7 @@ ivas_error ivas_td_binaural_renderer_unwrap( if ( subframe_idx == ISM_METADATA_DELAY_SUBFRAME ) { /* Update object position(s) */ - TDREND_Update_object_positions( hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, output ); + TDREND_Update_object_positions( hBinRendererTd, num_src, ivas_format, hIsmMetaData ); } #endif /* Update the listener's location/orientation */ @@ -471,21 +471,28 @@ static void TDREND_Clear_Update_flags( void TDREND_Update_object_positions( BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o : TD Renderer handle */ const int16_t num_src, /* i : number of sources to render */ - const int16_t lfe_idx, /* i : Input LFE index */ - const IVAS_FORMAT in_format, /* i : Format of input sources */ - const ISM_METADATA_HANDLE *hIsmMetaData, /* i : Input metadata for ISM objects */ +#ifndef FIX_356_ISM_METADATA_SYNC + const int16_t lfe_idx, /* i : Input LFE index */ +#endif + const IVAS_FORMAT in_format, /* i : Format of input sources */ + const ISM_METADATA_HANDLE *hIsmMetaData /* i : Input metadata for ISM objects */ +#ifndef FIX_356_ISM_METADATA_SYNC + , #ifdef JBM_TSM_ON_TCS float *output[] #else float output[][L_FRAME48k] /* i/o: SCE/MC channels */ #endif +#endif ) { TDREND_DirAtten_t *DirAtten_p; int16_t nS; float Pos[3]; float Dir[3]; +#ifndef FIX_356_ISM_METADATA_SYNC int16_t c_indx; +#endif DirAtten_p = hBinRendererTd->DirAtten_p; @@ -661,8 +668,8 @@ ivas_error ivas_td_binaural_renderer_ext( const IVAS_REND_HeadRotData *headRotData, /* i : Input head positions */ const IVAS_REND_AudioObjectPosition *currentPos, /* i : Object position */ const REVERB_HANDLE hReverb, /* i : Reverberator handle */ - const int16_t output_frame, /* i : output frame length */ - float output[][L_FRAME48k] /* i/o: SCE channels / Binaural synthesis */ + const int16_t output_frame, /* i : output frame length */ + float output[][L_FRAME48k] /* i/o: SCE channels / Binaural synthesis */ ) { ISM_METADATA_FRAME hIsmMetaDataFrame; diff --git a/lib_rend/ivas_prot_rend.h b/lib_rend/ivas_prot_rend.h index b43a048f38..2c1ac1394b 100644 --- a/lib_rend/ivas_prot_rend.h +++ b/lib_rend/ivas_prot_rend.h @@ -311,14 +311,19 @@ void TDREND_Update_listener_orientation( void TDREND_Update_object_positions( BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o : TD Renderer handle */ const int16_t num_src, /* i : number of sources to render */ +#ifndef FIX_356_ISM_METADATA_SYNC const int16_t lfe_idx, /* i : Input LFE index */ +#endif const IVAS_FORMAT in_format, /* i : Format of input sources */ - const ISM_METADATA_HANDLE *hIsmMetaData, /* i : Input metadata for ISM objects */ + const ISM_METADATA_HANDLE *hIsmMetaData /* i : Input metadata for ISM objects */ +#ifndef FIX_356_ISM_METADATA_SYNC + , #ifdef JBM_TSM_ON_TCS float *output[] /* i/o: SCE/MC channels */ #else float output[][L_FRAME48k] /* i/o: SCE/MC channels */ #endif +#endif ); void BSplineModelEvalDealloc( -- GitLab From 8fbd3627ff5189474e3249d59ff0d84fcd41510b Mon Sep 17 00:00:00 2001 From: Sumeyra Kanik Date: Wed, 17 May 2023 15:17:46 +0200 Subject: [PATCH 08/18] Keep subframe update idx as a variable, not constant --- lib_com/ivas_cnst.h | 3 --- lib_dec/ivas_objectRenderer_internal.c | 8 +++++++- lib_rend/ivas_objectRenderer.c | 25 +++++++++++++++++++++---- lib_rend/ivas_objectRenderer_sources.c | 6 +++++- lib_rend/ivas_prot_rend.h | 8 ++++++++ 5 files changed, 41 insertions(+), 9 deletions(-) diff --git a/lib_com/ivas_cnst.h b/lib_com/ivas_cnst.h index 57052ad950..f6369235a6 100755 --- a/lib_com/ivas_cnst.h +++ b/lib_com/ivas_cnst.h @@ -364,9 +364,6 @@ typedef enum #endif #define ISM_EXTENDED_METADATA_BITS 1 #define ISM_METADATA_RS_MAX_FRAMES 5 /* Number of frames with opposite extended metadata flags before switching */ -#ifdef FIX_356_ISM_METADATA_SYNC -#define ISM_METADATA_DELAY_SUBFRAME 2 /* Number of subframes to delay metadata to sync with audio */ -#endif /* Parametric ISM */ #define MAX_PARAM_ISM_NBANDS 11 diff --git a/lib_dec/ivas_objectRenderer_internal.c b/lib_dec/ivas_objectRenderer_internal.c index 6d701be0a5..47f950a550 100644 --- a/lib_dec/ivas_objectRenderer_internal.c +++ b/lib_dec/ivas_objectRenderer_internal.c @@ -102,6 +102,9 @@ void ObjRenderIVASSubframe( float *output_f_local[BINAURAL_CHANNELS]; float *tc_local[MAX_TRANSPORT_CHANNELS]; int16_t ch, slot_size, slots_to_render, output_frame; +#ifdef FIX_356_ISM_METADATA_SYNC + int16_t subframe_update = 0; +#endif for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) { p_reverb_signal[ch] = reverb_signal[ch]; @@ -149,8 +152,11 @@ void ObjRenderIVASSubframe( } /* Render subframe */ +#ifdef FIX_356_ISM_METADATA_SYNC + TDREND_GetMix( st_ivas->hBinRendererTd, output_f_local, output_frame, 0, subframe_update ); +#else TDREND_GetMix( st_ivas->hBinRendererTd, output_f_local, output_frame, 0 ); - +#endif if ( st_ivas->hRenderConfig != NULL ) /* Renderer Configuration not enabled in TD standalone renderer */ { if ( st_ivas->hRenderConfig->roomAcoustics.late_reverb_on ) diff --git a/lib_rend/ivas_objectRenderer.c b/lib_rend/ivas_objectRenderer.c index a7a8df5eed..6214f74670 100644 --- a/lib_rend/ivas_objectRenderer.c +++ b/lib_rend/ivas_objectRenderer.c @@ -290,7 +290,8 @@ ivas_error ivas_td_binaural_renderer_unwrap( subframe_length = output_frame / MAX_PARAM_SPATIAL_SUBFRAMES; #ifdef FIX_356_ISM_METADATA_SYNC - int16_t c_indx, nS; + int16_t c_indx, nS, subframe_update; + subframe_update = 2; c_indx = 0; for ( nS = 0; nS < num_src; nS++ ) @@ -310,7 +311,7 @@ ivas_error ivas_td_binaural_renderer_unwrap( for ( subframe_idx = 0; subframe_idx < MAX_PARAM_SPATIAL_SUBFRAMES; subframe_idx++ ) { #ifdef FIX_356_ISM_METADATA_SYNC - if ( subframe_idx == ISM_METADATA_DELAY_SUBFRAME ) + if ( subframe_idx == subframe_update ) { /* Update object position(s) */ TDREND_Update_object_positions( hBinRendererTd, num_src, ivas_format, hIsmMetaData ); @@ -332,7 +333,11 @@ ivas_error ivas_td_binaural_renderer_unwrap( } /* Render subframe */ +#ifdef FIX_356_ISM_METADATA_SYNC + if ( ( error = TDREND_GetMix( hBinRendererTd, output, subframe_length, subframe_idx, subframe_update ) ) != IVAS_ERR_OK ) +#else if ( ( error = TDREND_GetMix( hBinRendererTd, output, subframe_length, subframe_idx ) ) != IVAS_ERR_OK ) +#endif { return error; } @@ -365,6 +370,10 @@ ivas_error TDREND_GetMix( #endif const int16_t subframe_length, /* i/o: subframe length */ const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ +#ifdef FIX_356_ISM_METADATA_SYNC + , + const int16_t subframe_update +#endif ) { int16_t i; @@ -379,7 +388,10 @@ ivas_error TDREND_GetMix( #ifdef ISM_NON_DIEGETIC_PAN float pan_left, pan_right; #endif - +#ifdef FIX_356_ISM_METADATA_SYNC + int16_t subframe_update_flag; + subframe_update_flag = subframe_idx == subframe_update; +#endif error = IVAS_ERR_OK; /* Clear the output buffer to accumulate rendered sources */ @@ -408,7 +420,12 @@ ivas_error TDREND_GetMix( { TDREND_SRC_REND_UpdateFiltersFromSpatialParams( hBinRendererTd, SrcRend_p, SrcSpatial_p, Src_p->hrf_left_prev, Src_p->hrf_right_prev, hrf_left_delta, hrf_right_delta, &intp_count, &Src_p->filterlength, &Src_p->itd, &Src_p->Gain, Src_p, - subframe_idx ); +#ifdef FIX_356_ISM_METADATA_SYNC + subframe_update_flag +#else + subframe_idx +#endif + ); } /* Render source if needed */ diff --git a/lib_rend/ivas_objectRenderer_sources.c b/lib_rend/ivas_objectRenderer_sources.c index f622764482..35f8f90668 100644 --- a/lib_rend/ivas_objectRenderer_sources.c +++ b/lib_rend/ivas_objectRenderer_sources.c @@ -259,7 +259,11 @@ void TDREND_SRC_REND_UpdateFiltersFromSpatialParams( int16_t *itd, /* o: ITD value */ float *Gain, /* o: Gain value */ TDREND_SRC_t *Src_p, /* i/o: Source pointer */ +#ifdef FIX_356_ISM_METADATA_SYNC + const int16_t subframe_update_flag +#else const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ +#endif ) { TDREND_MIX_Listener_t *Listener_p; @@ -354,7 +358,7 @@ void TDREND_SRC_REND_UpdateFiltersFromSpatialParams( *intp_count = min( MAX_INTERPOLATION_STEPS, max( (int16_t) ( fabsf( azim_delta ) * MAX_ANGULAR_STEP_INV ), (int16_t) ( fabsf( elev_delta ) * MAX_ANGULAR_STEP_INV ) ) ); #ifdef FIX_356_ISM_METADATA_SYNC - if ( ( *intp_count > 0 ) && subframe_idx == ISM_METADATA_DELAY_SUBFRAME ) + if ( ( *intp_count > 0 ) && subframe_update_flag ) #else if ( ( *intp_count > 0 ) && subframe_idx == 0 ) #endif diff --git a/lib_rend/ivas_prot_rend.h b/lib_rend/ivas_prot_rend.h index 2c1ac1394b..f8738a2c8a 100644 --- a/lib_rend/ivas_prot_rend.h +++ b/lib_rend/ivas_prot_rend.h @@ -299,6 +299,10 @@ ivas_error TDREND_GetMix( #endif const int16_t subframe_length, /* i/o: subframe length */ const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ +#ifdef FIX_356_ISM_METADATA_SYNC + , + const int16_t subframe_update /* Number of subframes to delay metadata to sync with audio */ +#endif ); void TDREND_Update_listener_orientation( @@ -394,7 +398,11 @@ void TDREND_SRC_REND_UpdateFiltersFromSpatialParams( int16_t *itd, /* o: ITD value */ float *Gain, /* o: Gain value */ TDREND_SRC_t *Src_p, +#ifdef FIX_356_ISM_METADATA_SYNC + const int16_t subframe_update_flag /* i : Flag to determine update subframe idx */ +#else const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ +#endif ); ivas_error TDREND_SRC_Alloc( -- GitLab From 5cc82a60fd4debbf99cea8463202f56c200a8e50 Mon Sep 17 00:00:00 2001 From: Sumeyra Kanik Date: Wed, 17 May 2023 15:40:40 +0200 Subject: [PATCH 09/18] Reverse subframe update to 0 --- lib_rend/ivas_objectRenderer.c | 791 ++++++++++++++++++++++++- lib_rend/ivas_objectRenderer_sources.c | 2 +- 2 files changed, 791 insertions(+), 2 deletions(-) diff --git a/lib_rend/ivas_objectRenderer.c b/lib_rend/ivas_objectRenderer.c index f36a229fd9..a69ca69c5c 100644 --- a/lib_rend/ivas_objectRenderer.c +++ b/lib_rend/ivas_objectRenderer.c @@ -291,7 +291,796 @@ ivas_error ivas_td_binaural_renderer_unwrap( #ifdef FIX_356_ISM_METADATA_SYNC int16_t c_indx, nS, subframe_update; - subframe_update = 2; + subframe_update = 0; + c_indx = 0; + + for ( nS = 0; nS < num_src; nS++ ) + { + if ( !( ivas_format == MC_FORMAT && nS == lfe_idx ) ) /* Skip LFE for MC */ + { + hBinRendererTd->Sources[c_indx]->InputFrame_p = output[nS]; + hBinRendererTd->Sources[c_indx]->SrcRend_p->InputAvailable = TRUE; + c_indx++; + } + } +#else + /* Update object position(s) */ + TDREND_Update_object_positions( hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, output ); +#endif + + for ( subframe_idx = 0; subframe_idx < MAX_PARAM_SPATIAL_SUBFRAMES; subframe_idx++ ) + { +#ifdef FIX_356_ISM_METADATA_SYNC + if ( subframe_idx == subframe_update ) + { + /* Update object position(s) */ + TDREND_Update_object_positions( hBinRendererTd, num_src, ivas_format, hIsmMetaData ); + } +#endif + /* Update the listener's location/orientation */ + TDREND_Update_listener_orientation( hBinRendererTd, Opt_Headrotation, ( Quaternions != NULL ) ? &Quaternions[subframe_idx] : NULL, ( Pos != NULL ) ? &Pos[subframe_idx] : NULL ); + + if ( hReverb != NULL && hReverb->pConfig.roomAcoustics.late_reverb_on ) + { +#ifdef JBM_TSM_ON_TCS + if ( ( error = ivas_reverb_process( hReverb, transport_config, 0, output, p_reverb_signal, subframe_idx ) ) != IVAS_ERR_OK ) +#else + if ( ( error = ivas_reverb_process( hReverb, transport_config, 0, output, reverb_signal, subframe_idx ) ) != IVAS_ERR_OK ) +#endif + { + return error; + } + } + + /* Render subframe */ +#ifdef FIX_356_ISM_METADATA_SYNC + if ( ( error = TDREND_GetMix( hBinRendererTd, output, subframe_length, subframe_idx, subframe_update ) ) != IVAS_ERR_OK ) +#else + if ( ( error = TDREND_GetMix( hBinRendererTd, output, subframe_length, subframe_idx ) ) != IVAS_ERR_OK ) +#endif + { + return error; + } + } + + + if ( hReverb != NULL && hReverb->pConfig.roomAcoustics.late_reverb_on ) + { + /* add reverb to rendered signals */ + v_add( reverb_signal[0], output[0], output[0], output_frame ); + v_add( reverb_signal[1], output[1], output[1], output_frame ); + } + + return IVAS_ERR_OK; +} + + +/*---------------------------------------------------------------------* + * TDREND_GetMix() + * + * Render one 5 ms subframe from the mixer + *---------------------------------------------------------------------*/ + +ivas_error TDREND_GetMix( + BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD renderer handle */ +#ifdef JBM_TSM_ON_TCS + float *output[], /* i/o: ISM object synth / rendered output in 0,1 */ +#else + float output[][L_FRAME48k], /* i/o: ISM object synth / rendered output in 0,1 */ +#endif + const int16_t subframe_length, /* i/o: subframe length */ + const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ +#ifdef FIX_356_ISM_METADATA_SYNC + , + const int16_t subframe_update +#endif +) +{ + int16_t i; + TDREND_SRC_t *Src_p; + TDREND_SRC_SPATIAL_t *SrcSpatial_p; + TDREND_SRC_REND_t *SrcRend_p; + ivas_error error; + float output_buf[2][L_SPATIAL_SUBFR_48k]; /* Temp buffer for left/right rendered signal */ + float hrf_left_delta[SFX_SPAT_BIN_MAX_FILTER_LENGTH]; + float hrf_right_delta[SFX_SPAT_BIN_MAX_FILTER_LENGTH]; + int16_t intp_count; +#ifdef ISM_NON_DIEGETIC_PAN + float pan_left, pan_right; +#endif +#ifdef FIX_356_ISM_METADATA_SYNC + int16_t subframe_update_flag; + subframe_update_flag = subframe_idx == subframe_update; +#endif + error = IVAS_ERR_OK; + + /* Clear the output buffer to accumulate rendered sources */ + set_f( output_buf[0], 0.0f, subframe_length ); + set_f( output_buf[1], 0.0f, subframe_length ); + + /* Clear interpolation buffers and counter */ + set_f( hrf_left_delta, 0.0f, SFX_SPAT_BIN_MAX_FILTER_LENGTH ); + set_f( hrf_right_delta, 0.0f, SFX_SPAT_BIN_MAX_FILTER_LENGTH ); + intp_count = 0; + + /* Create the mix */ + /* Loop through the source list and render each source */ + for ( i = 0; i < hBinRendererTd->NumOfSrcs; i++ ) + { + Src_p = hBinRendererTd->Sources[i]; + SrcSpatial_p = Src_p->SrcSpatial_p; + SrcRend_p = Src_p->SrcRend_p; + + /* Update rendering params if needed */ +#ifdef ISM_NON_DIEGETIC_PAN + if ( ( SrcRend_p->PlayStatus == TDREND_PLAYSTATUS_PLAYING ) && ( hBinRendererTd->Listener_p->PoseUpdated || SrcSpatial_p->Updated ) ) +#else + if ( hBinRendererTd->Listener_p->PoseUpdated || SrcSpatial_p->Updated ) +#endif + { + TDREND_SRC_REND_UpdateFiltersFromSpatialParams( hBinRendererTd, SrcRend_p, SrcSpatial_p, Src_p->hrf_left_prev, + Src_p->hrf_right_prev, hrf_left_delta, hrf_right_delta, &intp_count, &Src_p->filterlength, &Src_p->itd, &Src_p->Gain, Src_p, +#ifdef FIX_356_ISM_METADATA_SYNC + subframe_update_flag +#else + subframe_idx +#endif + ); + } + + /* Render source if needed */ + if ( ( SrcRend_p->InputAvailable == TRUE ) && ( SrcRend_p->PlayStatus == TDREND_PLAYSTATUS_PLAYING ) ) + { + error = TDREND_REND_RenderSourceHRFilt( Src_p, hrf_left_delta, hrf_right_delta, intp_count, output_buf, subframe_length ); + } + +#ifdef ISM_NON_DIEGETIC_PAN + if ( ( SrcRend_p->InputAvailable == TRUE ) && ( SrcRend_p->PlayStatus == TDREND_PLAYSTATUS_PLAYING_NON_DIEGETIC ) ) + { + pan_left = ( SrcSpatial_p->Pos_p[1] + 1.f ) * 0.5f; + pan_right = 1.f - pan_left; + v_multc_acc( &Src_p->InputFrame_p[subframe_idx * subframe_length], pan_left, output_buf[0], subframe_length ); + v_multc_acc( &Src_p->InputFrame_p[subframe_idx * subframe_length], pan_right, output_buf[1], subframe_length ); + } +#endif + } + + /* Populate output variable */ + mvr2r( output_buf[0], output[0] + subframe_idx * subframe_length, subframe_length ); /* Left */ + mvr2r( output_buf[1], output[1] + subframe_idx * subframe_length, subframe_length ); /* Right */ + + /* Clear the PoseUpdated and Source position update flags */ + TDREND_Clear_Update_flags( hBinRendererTd ); + + return error; +} + + +/*---------------------------------------------------------------------* + * TDREND_Clear_Update_flags() + * + * Initializes the audio mixer module + *---------------------------------------------------------------------*/ + +static void TDREND_Clear_Update_flags( + BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd /* i/o: TD renderer handle */ +) +{ + int16_t i; + + hBinRendererTd->Listener_p->PoseUpdated = FALSE; + + for ( i = 0; i < hBinRendererTd->NumOfSrcs; i++ ) + { + hBinRendererTd->Sources[i]->SrcSpatial_p->Updated = FALSE; + } + + return; +} + + +/*---------------------------------------------------------------------* + * TDREND_Update_object_positions() + * + * Update object position(s) + *---------------------------------------------------------------------*/ + +void TDREND_Update_object_positions( + BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o : TD Renderer handle */ + const int16_t num_src, /* i : number of sources to render */ +#ifndef FIX_356_ISM_METADATA_SYNC + const int16_t lfe_idx, /* i : Input LFE index */ +#endif + const IVAS_FORMAT in_format, /* i : Format of input sources */ + const ISM_METADATA_HANDLE *hIsmMetaData /* i : Input metadata for ISM objects */ +#ifndef FIX_356_ISM_METADATA_SYNC + , +#ifdef JBM_TSM_ON_TCS + float *output[] +#else + float output[][L_FRAME48k] /* i/o: SCE/MC channels */ +#endif +#endif +) +{ + TDREND_DirAtten_t *DirAtten_p; + int16_t nS; + float Pos[3]; + float Dir[3]; +#ifndef FIX_356_ISM_METADATA_SYNC + int16_t c_indx; +#endif + + DirAtten_p = hBinRendererTd->DirAtten_p; + + /* For each source, write the frame data to the source object*/ +#ifndef FIX_356_ISM_METADATA_SYNC + c_indx = 0; +#endif + for ( nS = 0; nS < num_src; nS++ ) + { +#ifndef FIX_356_ISM_METADATA_SYNC + if ( !( in_format == MC_FORMAT && nS == lfe_idx ) ) /* Skip LFE for MC */ + { + hBinRendererTd->Sources[c_indx]->InputFrame_p = output[nS]; + hBinRendererTd->Sources[c_indx]->SrcRend_p->InputAvailable = TRUE; + c_indx++; + } +#endif + if ( in_format == ISM_FORMAT ) + { + /* Update the source positions */ + /* Source position and direction */ + angles_to_vec( hIsmMetaData[nS]->radius, hIsmMetaData[nS]->azimuth, hIsmMetaData[nS]->elevation, Pos ); + angles_to_vec( 1.0f, hIsmMetaData[nS]->yaw, hIsmMetaData[nS]->pitch, Dir ); + +#ifndef FIX_463_TD_RENDERER_DIRECTIVITY_RESET + + /* Source directivity info */ + DirAtten_p->ConeInnerAngle = 360.0f; + DirAtten_p->ConeOuterAngle = 360.0f; + DirAtten_p->ConeOuterGain = 1.0f; +#endif + + TDREND_MIX_SRC_SetPos( hBinRendererTd, nS, Pos ); + TDREND_MIX_SRC_SetDirAtten( hBinRendererTd, nS, DirAtten_p ); + +#ifdef ISM_NON_DIEGETIC_PAN + if ( hIsmMetaData[nS]->non_diegetic_flag ) + { + Pos[0] = 0; + Pos[1] = hIsmMetaData[nS]->azimuth / 90.f; + Pos[2] = 0; + TDREND_MIX_SRC_SetPos( hBinRendererTd, nS, Pos ); + TDREND_MIX_SRC_SetPlayState( hBinRendererTd, nS, TDREND_PLAYSTATUS_PLAYING_NON_DIEGETIC ); + } + else + { + TDREND_MIX_SRC_SetPlayState( hBinRendererTd, nS, TDREND_PLAYSTATUS_PLAYING ); + } +#else + TDREND_MIX_SRC_SetPlayState( hBinRendererTd, nS, TDREND_PLAYSTATUS_PLAYING ); +#endif + TDREND_MIX_SRC_SetDir( hBinRendererTd, nS, Dir ); + } + } + + return; +} + + +/*---------------------------------------------------------------------* + * TDREND_Update_listener_orientation() + * + * Update listener orientation (s) + *---------------------------------------------------------------------*/ + +void TDREND_Update_listener_orientation( + BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD Renderer handle */ + const int16_t headRotEnabled, /* i : Headrotation flag */ + const IVAS_QUATERNION *headPosition, /* i : Listener orientation */ + const IVAS_VECTOR3 *Pos /* i : Listener Position */ +) +{ + float FrontVec[3]; + float UpVec[3]; + float Rmat[3][3]; + float Pos_p[3]; + + if ( headRotEnabled ) + { + /* Obtain head rotation matrix */ + QuatToRotMat( *headPosition, Rmat ); + /* Apply rotation matrix to looking vector [1;0;0] */ + FrontVec[0] = Rmat[0][0]; + FrontVec[1] = Rmat[0][1]; + FrontVec[2] = Rmat[0][2]; + /* Apply rotation matrix to up vector [0;0;1] */ + UpVec[0] = Rmat[2][0]; + UpVec[1] = Rmat[2][1]; + UpVec[2] = Rmat[2][2]; + /* Input position */ + Pos_p[0] = ( *Pos ).x; + Pos_p[1] = ( *Pos ).y; + Pos_p[2] = ( *Pos ).z; + } + else + { + /* Oriented with looking vector along the x axis */ + FrontVec[0] = 1.0f; + FrontVec[1] = 0.0f; + FrontVec[2] = 0.0f; + /* Oriented with up vector along the z axis */ + UpVec[0] = 0.0f; + UpVec[1] = 0.0f; + UpVec[2] = 1.0f; + /* Listener at the origin */ + Pos_p[0] = 0.0f; + Pos_p[1] = 0.0f; + Pos_p[2] = 0.0f; + } + + /* Set the listener position and orientation:*/ + TDREND_MIX_LIST_SetPos( hBinRendererTd, Pos_p ); + TDREND_MIX_LIST_SetOrient( hBinRendererTd, FrontVec, UpVec ); + + return; +} + + +/*---------------------------------------------------------------------* + * ivas_td_binaural_open_ext() + * + * + *---------------------------------------------------------------------*/ + +ivas_error ivas_td_binaural_open_ext( + TDREND_WRAPPER *pTDRend, + IVAS_REND_AudioConfig inConfig, + RENDER_CONFIG_DATA *hRendCfg, /* i : Renderer configuration */ + LSSETUP_CUSTOM_STRUCT *customLsInput, + const int32_t outFs ) +{ + int16_t nchan_transport; + AUDIO_CONFIG transport_config; + IVAS_FORMAT ivas_format; + IVAS_OUTPUT_SETUP hTransSetup; + ivas_error error; + float *directivity = NULL; + + if ( inConfig != IVAS_REND_AUDIO_CONFIG_LS_CUSTOM ) + { + if ( ( error = getAudioConfigNumChannels( inConfig, &nchan_transport ) ) != IVAS_ERR_OK ) + { + return error; + } + } + else + { + nchan_transport = customLsInput->num_spk; + } + + transport_config = getIvasAudioConfigFromRendAudioConfig( inConfig ); + ivas_format = ( getAudioConfigType( inConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) ? MC_FORMAT : ISM_FORMAT; + hTransSetup.ls_azimuth = customLsInput->ls_azimuth; + hTransSetup.ls_elevation = customLsInput->ls_elevation; + + if ( NULL != hRendCfg ) + { + directivity = hRendCfg->directivity; + } + + return ivas_td_binaural_open_unwrap( &pTDRend->hHrtfTD, outFs, nchan_transport, ivas_format, transport_config, directivity, hTransSetup, &pTDRend->hBinRendererTd, &pTDRend->binaural_latency_ns ); +} + + +/*---------------------------------------------------------------------* + * ivas_td_binaural_renderer_ext() + * + * Receives the current frames for the object streams, updates metadata + * and renders the current frame. + *---------------------------------------------------------------------*/ + +ivas_error ivas_td_binaural_renderer_ext( + const TDREND_WRAPPER *pTDRend, /* i : TD Renderer wrapper structure */ + const IVAS_REND_AudioConfig inConfig, /* i : Input audio configuration */ + const LSSETUP_CUSTOM_STRUCT *customLsInput, /* i : Input custom loudspeaker layout */ + const IVAS_REND_HeadRotData *headRotData, /* i : Input head positions */ + const IVAS_REND_AudioObjectPosition *currentPos, /* i : Object position */ + const REVERB_HANDLE hReverb, /* i : Reverberator handle */ + const int16_t output_frame, /* i : output frame length */ + float output[][L_FRAME48k] /* i/o: SCE channels / Binaural synthesis */ +) +{ + ISM_METADATA_FRAME hIsmMetaDataFrame; + ISM_METADATA_HANDLE hIsmMetaData[1]; + int16_t lfe_idx; + int16_t num_src; + IVAS_FORMAT ivas_format; + IVAS_REND_AudioConfigType inConfigType; + AUDIO_CONFIG transport_config; + ivas_error error; +#ifdef JBM_TSM_ON_TCS + float *p_output[MAX_OUTPUT_CHANNELS]; + int16_t ch; + + for ( ch = 0; ch < MAX_OUTPUT_CHANNELS; ch++ ) + { + p_output[ch] = output[ch]; + } +#endif + + push_wmops( "ivas_td_binaural_renderer_ext" ); + + inConfigType = getAudioConfigType( inConfig ); + lfe_idx = LFE_CHANNEL; + hIsmMetaData[0] = NULL; + + if ( inConfigType == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) + { + ivas_format = MC_FORMAT; + transport_config = getIvasAudioConfigFromRendAudioConfig( inConfig ); + if ( inConfig != IVAS_REND_AUDIO_CONFIG_LS_CUSTOM ) + { + if ( ( error = getAudioConfigNumChannels( inConfig, &num_src ) ) != IVAS_ERR_OK ) + { + return error; + } + } + else + { + lfe_idx = ( customLsInput->num_lfe > 0 ) ? customLsInput->lfe_idx[0] : -1; + num_src = customLsInput->num_spk + customLsInput->num_lfe; + } + } + else + { + ivas_format = ISM_FORMAT; + num_src = 1; + transport_config = AUDIO_CONFIG_ISM1; + hIsmMetaData[0] = &hIsmMetaDataFrame; + hIsmMetaData[0]->azimuth = currentPos->azimuth; + hIsmMetaData[0]->elevation = currentPos->elevation; + hIsmMetaData[0]->yaw = currentPos->yaw; + hIsmMetaData[0]->pitch = currentPos->pitch; + hIsmMetaData[0]->radius = currentPos->radius; +#ifdef ISM_NON_DIEGETIC_PAN + hIsmMetaData[0]->non_diegetic_flag = currentPos->non_diegetic_flag; +#endif + } + +#ifdef JBM_TSM_ON_TCS + if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, + ( headRotData != NULL ) ? headRotData->headPositions : NULL, + ( headRotData != NULL ) ? headRotData->Pos : NULL, p_output, output_frame ) ) != IVAS_ERR_OK ) +#else + if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, + ( headRotData != NULL ) ? headRotData->headPositions : NULL, + ( headRotData != NULL ) ? headRotData->Pos : NULL, output, output_frame ) ) != IVAS_ERR_OK ) + +#endif + { + return error; + } + + pop_wmops(); + + return IVAS_ERR_OK; +} + +/*---------------------------------------------------------------------* + * angles_to_vec() + * + * Convert azimuth and elevation angles to position/orientation vector + *---------------------------------------------------------------------*/ + +static void angles_to_vec( + const float radius, /* i : radius */ + const float azimuth, /* i : Azimuth angle */ + const float elevation, /* i : Elevation angle */ + float *vec /* o : Pos/Dir vector */ +) +{ + vec[0] = radius * cosf( elevation * PI_OVER_180 ) * cosf( azimuth * PI_OVER_180 ); + vec[1] = radius * cosf( elevation * PI_OVER_180 ) * sinf( azimuth * PI_OVER_180 ); + vec[2] = radius * sinf( elevation * PI_OVER_180 ); + + return; +} +/****************************************************************************************************** + + (C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository. All Rights Reserved. + + This software is protected by copyright law and by international treaties. + The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository retain full ownership rights in their respective contributions in + the software. This notice grants no license of any kind, including but not limited to patent + license, nor is any license granted by implication, estoppel or otherwise. + + Contributors are required to enter into the IVAS codec Public Collaboration agreement before making + contributions. + + This software is provided "AS IS", without any express or implied warranties. The software is in the + development stage. It is intended exclusively for experts who have experience with such software and + solely for the purpose of inspection. All implied warranties of non-infringement, merchantability + and fitness for a particular purpose are hereby disclaimed and excluded. + + Any dispute, controversy or claim arising under or in relation to providing this software shall be + submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in + accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and + the United Nations Convention on Contracts on the International Sales of Goods. + +*******************************************************************************************************/ + +#include "ivas_stat_rend.h" +#include +#include "options.h" +#include "prot.h" +#ifdef ISM_NON_DIEGETIC_PAN +#include "ivas_prot.h" +#endif +#include "ivas_prot_rend.h" +#include +#include "ivas_rom_com.h" +#ifdef DEBUGGING +#include "debug.h" +#endif +#include "wmc_auto.h" + + +/*---------------------------------------------------------------------* + * Local function prototypes + *---------------------------------------------------------------------*/ + + +static void TDREND_Clear_Update_flags( BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd ); + +static void angles_to_vec( const float radius, const float azimuth, const float elevation, float *vec ); + + +/*---------------------------------------------------------------------* + * ivas_td_binaural_open_unwrap() + * + * Call TD open/init function without st_ivas + *---------------------------------------------------------------------*/ + +ivas_error ivas_td_binaural_open_unwrap( + TDREND_HRFILT_FiltSet_t **hHrtfTD, /* i/o: HR filter model (from file or NULL) */ + const int32_t output_Fs, /* i : Output sampling rate */ + const int16_t nchan_transport, /* i : Number of channels */ + const IVAS_FORMAT ivas_format, /* i : IVAS format (ISM/MC) */ + const AUDIO_CONFIG transport_config, /* i : Transport configuration */ + const float *directivity, /* i : Directivity pattern (used for ISM) */ + const IVAS_OUTPUT_SETUP hTransSetup, /* i : Loudspeaker layout */ + BINAURAL_TD_OBJECT_RENDERER_HANDLE *hBinRendererTd, /* o : TD renderer handle */ + int32_t *binaural_latency_ns /* i : Binauralization delay */ +) +{ + BINAURAL_TD_OBJECT_RENDERER_HANDLE pBinRendTd; + TDREND_PosType_t PosType; + int16_t nS; + int16_t SrcInd[MAX_NUM_TDREND_CHANNELS]; + const float *ls_azimuth, *ls_elevation; + float Pos[3]; + float Dir[3]; + TDREND_DirAtten_t *DirAtten_p; + int16_t nchan_rend; + ivas_error error; + + error = IVAS_ERR_OK; + + if ( ( pBinRendTd = malloc( sizeof( BINAURAL_TD_OBJECT_RENDERER ) ) ) == NULL ) + { + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for TD renderer\n" ) ); + } + if ( ( pBinRendTd->TdRend_MixSpatSpec_p = malloc( sizeof( TDREND_MixSpatSpec_t ) ) ) == NULL ) + { + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for TD renderer\n" ) ); + } + if ( ( pBinRendTd->DirAtten_p = malloc( sizeof( TDREND_DirAtten_t ) ) ) == NULL ) + { + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for TD renderer\n" ) ); + } + if ( ( pBinRendTd->Listener_p = malloc( sizeof( TDREND_MIX_Listener_t ) ) ) == NULL ) + { + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for TD renderer\n" ) ); + } + + pBinRendTd->NumOfSrcs = 0; + pBinRendTd->MaxSrcInd = -1; + + /* Mixer spatial setup */ + pBinRendTd->TdRend_MixSpatSpec_p->UseCommonDistAttenModel = TRUE; + pBinRendTd->TdRend_MixSpatSpec_p->DistAttenModel = 0; /* 0=Turned off, else use TDREND_DIST_ATTEN_MODEL_INV_DIST_CLAMPED */ + + if ( ( error = TDREND_MIX_Init( pBinRendTd, hHrtfTD, pBinRendTd->TdRend_MixSpatSpec_p, output_Fs ) ) != IVAS_ERR_OK ) + { + return error; + } + + /* Set the attenuation (or can set MixSpatSpec.DistAttenModel above) */ + if ( ( error = TDREND_MIX_SetDistAttenModel( pBinRendTd, TDREND_DIST_ATTEN_MODEL_INV_DIST_CLAMPED ) ) != IVAS_ERR_OK ) + { + return error; + } + + /* Add sources to module and mixer, headphones */ + PosType = TDREND_POSTYPE_ABSOLUTE; /* or TDREND_POSTYPE_RELATIVE_TO_LISTENER */ + + nchan_rend = nchan_transport; + if ( ( ivas_format == MC_FORMAT ) && ( transport_config != AUDIO_CONFIG_LS_CUSTOM ) ) + { + nchan_rend--; /* Skip LFE channel -- added to the others */ + } + + for ( nS = 0; nS < nchan_rend; nS++ ) + { + if ( ( error = TDREND_MIX_AddSrc( pBinRendTd, &SrcInd[nS], PosType ) ) != IVAS_ERR_OK ) + { + return error; + } + } + + if ( ivas_format == MC_FORMAT ) + { + switch ( transport_config ) + { + case AUDIO_CONFIG_5_1: + ls_azimuth = ls_azimuth_CICP6; + ls_elevation = ls_elevation_CICP6; + break; + case AUDIO_CONFIG_7_1: + ls_azimuth = ls_azimuth_CICP12; + ls_elevation = ls_elevation_CICP12; + break; + case AUDIO_CONFIG_5_1_2: + ls_azimuth = ls_azimuth_CICP14; + ls_elevation = ls_elevation_CICP14; + break; + case AUDIO_CONFIG_5_1_4: + ls_azimuth = ls_azimuth_CICP16; + ls_elevation = ls_elevation_CICP16; + break; + case AUDIO_CONFIG_7_1_4: + ls_azimuth = ls_azimuth_CICP19; + ls_elevation = ls_elevation_CICP19; + break; + case AUDIO_CONFIG_LS_CUSTOM: + ls_azimuth = hTransSetup.ls_azimuth; + ls_elevation = hTransSetup.ls_elevation; + break; + default: + ls_azimuth = NULL; + ls_elevation = NULL; + } + + DirAtten_p = pBinRendTd->DirAtten_p; + + for ( nS = 0; nS < nchan_rend; nS++ ) + { + /* Set source positions according to loudspeaker layout */ + angles_to_vec( 1.0f, ls_azimuth[nS], ls_elevation[nS], Pos ); + Dir[0] = 1.0f; + Dir[1] = 0.0f; + Dir[2] = 0.0f; + + /* Source directivity info */ + DirAtten_p->ConeInnerAngle = 360.0f; + DirAtten_p->ConeOuterAngle = 360.0f; + DirAtten_p->ConeOuterGain = 1.0f; + + TDREND_MIX_SRC_SetPos( pBinRendTd, nS, Pos ); + TDREND_MIX_SRC_SetDir( pBinRendTd, nS, Dir ); + TDREND_MIX_SRC_SetPlayState( pBinRendTd, nS, TDREND_PLAYSTATUS_PLAYING ); + TDREND_MIX_SRC_SetDirAtten( pBinRendTd, nS, DirAtten_p ); + } + } + if ( ivas_format == ISM_FORMAT ) + { + DirAtten_p = pBinRendTd->DirAtten_p; + if ( NULL == directivity ) + { + DirAtten_p->ConeInnerAngle = 360.0f; /* Front cone */ + DirAtten_p->ConeOuterAngle = 360.0f; /* Back cone */ + DirAtten_p->ConeOuterGain = 1.0f; /* Back attenuation */ + } + else + { + DirAtten_p->ConeInnerAngle = directivity[0]; + DirAtten_p->ConeOuterAngle = directivity[1]; + DirAtten_p->ConeOuterGain = directivity[2]; + } + + for ( nS = 0; nS < nchan_rend; nS++ ) + { + TDREND_MIX_SRC_SetDirAtten( pBinRendTd, nS, DirAtten_p ); + } + } + + *hBinRendererTd = pBinRendTd; + *binaural_latency_ns = (int32_t) ( ( *hBinRendererTd )->HrFiltSet_p->latency_s * 1000000000.f ); + + return error; +} + + +/*---------------------------------------------------------------------* + * ivas_td_binaural_close() + * + * Close TD Object binaural renderer + *---------------------------------------------------------------------*/ + +void ivas_td_binaural_close( + BINAURAL_TD_OBJECT_RENDERER_HANDLE *hBinRendererTd /* i/o: TD binaural object renderer handle */ +) +{ + if ( hBinRendererTd == NULL || *hBinRendererTd == NULL ) + { + return; + } + + free( ( *hBinRendererTd )->TdRend_MixSpatSpec_p ); + free( ( *hBinRendererTd )->DirAtten_p ); + + TDREND_MIX_Dealloc( *hBinRendererTd ); + + free( *hBinRendererTd ); + *hBinRendererTd = NULL; + + return; +} + +/*---------------------------------------------------------------------* + * ivas_td_binaural_renderer_unwrap() + * + * Call ivas_td_binaural_renderer() without st_ivas. + *---------------------------------------------------------------------*/ + +ivas_error ivas_td_binaural_renderer_unwrap( + const REVERB_HANDLE hReverb, /* i : Reverberator handle */ + const AUDIO_CONFIG transport_config, /* i : Transport configuration */ + BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD binaural object renderer handle */ + const int16_t num_src, /* i : number of sources to render */ + const int16_t lfe_idx, /* i : LFE channel index */ + const IVAS_FORMAT ivas_format, /* i : IVAS format */ + ISM_METADATA_HANDLE *hIsmMetaData, /* i : ISM metadata handle */ + const int16_t Opt_Headrotation, /* i : Head rotation flag */ + const IVAS_QUATERNION *Quaternions, /* i : Head tracking data per subframe */ + const IVAS_VECTOR3 *Pos, /* i : Listener position data per subframe */ +#ifdef JBM_TSM_ON_TCS + float *output[], /* i/o: SCE channels / Binaural synthesis */ +#else + float output[][L_FRAME48k], /* i/o: SCE channels / Binaural synthesis */ +#endif + const int16_t output_frame /* i : output frame length */ +) +{ + int16_t subframe_length; + int16_t subframe_idx; + float reverb_signal[BINAURAL_CHANNELS][L_FRAME48k]; + ivas_error error; +#ifdef JBM_TSM_ON_TCS + float *p_reverb_signal[BINAURAL_CHANNELS]; + int16_t ch; + + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + p_reverb_signal[ch] = reverb_signal[ch]; + } +#endif + + subframe_length = output_frame / MAX_PARAM_SPATIAL_SUBFRAMES; + +#ifdef FIX_356_ISM_METADATA_SYNC + int16_t c_indx, nS, subframe_update; + subframe_update = 0; c_indx = 0; for ( nS = 0; nS < num_src; nS++ ) diff --git a/lib_rend/ivas_objectRenderer_sources.c b/lib_rend/ivas_objectRenderer_sources.c index 35f8f90668..a565bc08b5 100644 --- a/lib_rend/ivas_objectRenderer_sources.c +++ b/lib_rend/ivas_objectRenderer_sources.c @@ -262,7 +262,7 @@ void TDREND_SRC_REND_UpdateFiltersFromSpatialParams( #ifdef FIX_356_ISM_METADATA_SYNC const int16_t subframe_update_flag #else - const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ + const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ #endif ) { -- GitLab From 96bf30fb66c6b1c318bdb3ec3e8849d0c510e3f8 Mon Sep 17 00:00:00 2001 From: Sumeyra Kanik Date: Wed, 17 May 2023 15:48:39 +0200 Subject: [PATCH 10/18] Fix crash Reverse subframe update to 0 --- lib_rend/ivas_objectRenderer.c | 789 --------------------------------- 1 file changed, 789 deletions(-) diff --git a/lib_rend/ivas_objectRenderer.c b/lib_rend/ivas_objectRenderer.c index a69ca69c5c..36e187870f 100644 --- a/lib_rend/ivas_objectRenderer.c +++ b/lib_rend/ivas_objectRenderer.c @@ -787,792 +787,3 @@ static void angles_to_vec( return; } -/****************************************************************************************************** - - (C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. - -*******************************************************************************************************/ - -#include "ivas_stat_rend.h" -#include -#include "options.h" -#include "prot.h" -#ifdef ISM_NON_DIEGETIC_PAN -#include "ivas_prot.h" -#endif -#include "ivas_prot_rend.h" -#include -#include "ivas_rom_com.h" -#ifdef DEBUGGING -#include "debug.h" -#endif -#include "wmc_auto.h" - - -/*---------------------------------------------------------------------* - * Local function prototypes - *---------------------------------------------------------------------*/ - - -static void TDREND_Clear_Update_flags( BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd ); - -static void angles_to_vec( const float radius, const float azimuth, const float elevation, float *vec ); - - -/*---------------------------------------------------------------------* - * ivas_td_binaural_open_unwrap() - * - * Call TD open/init function without st_ivas - *---------------------------------------------------------------------*/ - -ivas_error ivas_td_binaural_open_unwrap( - TDREND_HRFILT_FiltSet_t **hHrtfTD, /* i/o: HR filter model (from file or NULL) */ - const int32_t output_Fs, /* i : Output sampling rate */ - const int16_t nchan_transport, /* i : Number of channels */ - const IVAS_FORMAT ivas_format, /* i : IVAS format (ISM/MC) */ - const AUDIO_CONFIG transport_config, /* i : Transport configuration */ - const float *directivity, /* i : Directivity pattern (used for ISM) */ - const IVAS_OUTPUT_SETUP hTransSetup, /* i : Loudspeaker layout */ - BINAURAL_TD_OBJECT_RENDERER_HANDLE *hBinRendererTd, /* o : TD renderer handle */ - int32_t *binaural_latency_ns /* i : Binauralization delay */ -) -{ - BINAURAL_TD_OBJECT_RENDERER_HANDLE pBinRendTd; - TDREND_PosType_t PosType; - int16_t nS; - int16_t SrcInd[MAX_NUM_TDREND_CHANNELS]; - const float *ls_azimuth, *ls_elevation; - float Pos[3]; - float Dir[3]; - TDREND_DirAtten_t *DirAtten_p; - int16_t nchan_rend; - ivas_error error; - - error = IVAS_ERR_OK; - - if ( ( pBinRendTd = malloc( sizeof( BINAURAL_TD_OBJECT_RENDERER ) ) ) == NULL ) - { - return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for TD renderer\n" ) ); - } - if ( ( pBinRendTd->TdRend_MixSpatSpec_p = malloc( sizeof( TDREND_MixSpatSpec_t ) ) ) == NULL ) - { - return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for TD renderer\n" ) ); - } - if ( ( pBinRendTd->DirAtten_p = malloc( sizeof( TDREND_DirAtten_t ) ) ) == NULL ) - { - return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for TD renderer\n" ) ); - } - if ( ( pBinRendTd->Listener_p = malloc( sizeof( TDREND_MIX_Listener_t ) ) ) == NULL ) - { - return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for TD renderer\n" ) ); - } - - pBinRendTd->NumOfSrcs = 0; - pBinRendTd->MaxSrcInd = -1; - - /* Mixer spatial setup */ - pBinRendTd->TdRend_MixSpatSpec_p->UseCommonDistAttenModel = TRUE; - pBinRendTd->TdRend_MixSpatSpec_p->DistAttenModel = 0; /* 0=Turned off, else use TDREND_DIST_ATTEN_MODEL_INV_DIST_CLAMPED */ - - if ( ( error = TDREND_MIX_Init( pBinRendTd, hHrtfTD, pBinRendTd->TdRend_MixSpatSpec_p, output_Fs ) ) != IVAS_ERR_OK ) - { - return error; - } - - /* Set the attenuation (or can set MixSpatSpec.DistAttenModel above) */ - if ( ( error = TDREND_MIX_SetDistAttenModel( pBinRendTd, TDREND_DIST_ATTEN_MODEL_INV_DIST_CLAMPED ) ) != IVAS_ERR_OK ) - { - return error; - } - - /* Add sources to module and mixer, headphones */ - PosType = TDREND_POSTYPE_ABSOLUTE; /* or TDREND_POSTYPE_RELATIVE_TO_LISTENER */ - - nchan_rend = nchan_transport; - if ( ( ivas_format == MC_FORMAT ) && ( transport_config != AUDIO_CONFIG_LS_CUSTOM ) ) - { - nchan_rend--; /* Skip LFE channel -- added to the others */ - } - - for ( nS = 0; nS < nchan_rend; nS++ ) - { - if ( ( error = TDREND_MIX_AddSrc( pBinRendTd, &SrcInd[nS], PosType ) ) != IVAS_ERR_OK ) - { - return error; - } - } - - if ( ivas_format == MC_FORMAT ) - { - switch ( transport_config ) - { - case AUDIO_CONFIG_5_1: - ls_azimuth = ls_azimuth_CICP6; - ls_elevation = ls_elevation_CICP6; - break; - case AUDIO_CONFIG_7_1: - ls_azimuth = ls_azimuth_CICP12; - ls_elevation = ls_elevation_CICP12; - break; - case AUDIO_CONFIG_5_1_2: - ls_azimuth = ls_azimuth_CICP14; - ls_elevation = ls_elevation_CICP14; - break; - case AUDIO_CONFIG_5_1_4: - ls_azimuth = ls_azimuth_CICP16; - ls_elevation = ls_elevation_CICP16; - break; - case AUDIO_CONFIG_7_1_4: - ls_azimuth = ls_azimuth_CICP19; - ls_elevation = ls_elevation_CICP19; - break; - case AUDIO_CONFIG_LS_CUSTOM: - ls_azimuth = hTransSetup.ls_azimuth; - ls_elevation = hTransSetup.ls_elevation; - break; - default: - ls_azimuth = NULL; - ls_elevation = NULL; - } - - DirAtten_p = pBinRendTd->DirAtten_p; - - for ( nS = 0; nS < nchan_rend; nS++ ) - { - /* Set source positions according to loudspeaker layout */ - angles_to_vec( 1.0f, ls_azimuth[nS], ls_elevation[nS], Pos ); - Dir[0] = 1.0f; - Dir[1] = 0.0f; - Dir[2] = 0.0f; - - /* Source directivity info */ - DirAtten_p->ConeInnerAngle = 360.0f; - DirAtten_p->ConeOuterAngle = 360.0f; - DirAtten_p->ConeOuterGain = 1.0f; - - TDREND_MIX_SRC_SetPos( pBinRendTd, nS, Pos ); - TDREND_MIX_SRC_SetDir( pBinRendTd, nS, Dir ); - TDREND_MIX_SRC_SetPlayState( pBinRendTd, nS, TDREND_PLAYSTATUS_PLAYING ); - TDREND_MIX_SRC_SetDirAtten( pBinRendTd, nS, DirAtten_p ); - } - } - if ( ivas_format == ISM_FORMAT ) - { - DirAtten_p = pBinRendTd->DirAtten_p; - if ( NULL == directivity ) - { - DirAtten_p->ConeInnerAngle = 360.0f; /* Front cone */ - DirAtten_p->ConeOuterAngle = 360.0f; /* Back cone */ - DirAtten_p->ConeOuterGain = 1.0f; /* Back attenuation */ - } - else - { - DirAtten_p->ConeInnerAngle = directivity[0]; - DirAtten_p->ConeOuterAngle = directivity[1]; - DirAtten_p->ConeOuterGain = directivity[2]; - } - - for ( nS = 0; nS < nchan_rend; nS++ ) - { - TDREND_MIX_SRC_SetDirAtten( pBinRendTd, nS, DirAtten_p ); - } - } - - *hBinRendererTd = pBinRendTd; - *binaural_latency_ns = (int32_t) ( ( *hBinRendererTd )->HrFiltSet_p->latency_s * 1000000000.f ); - - return error; -} - - -/*---------------------------------------------------------------------* - * ivas_td_binaural_close() - * - * Close TD Object binaural renderer - *---------------------------------------------------------------------*/ - -void ivas_td_binaural_close( - BINAURAL_TD_OBJECT_RENDERER_HANDLE *hBinRendererTd /* i/o: TD binaural object renderer handle */ -) -{ - if ( hBinRendererTd == NULL || *hBinRendererTd == NULL ) - { - return; - } - - free( ( *hBinRendererTd )->TdRend_MixSpatSpec_p ); - free( ( *hBinRendererTd )->DirAtten_p ); - - TDREND_MIX_Dealloc( *hBinRendererTd ); - - free( *hBinRendererTd ); - *hBinRendererTd = NULL; - - return; -} - -/*---------------------------------------------------------------------* - * ivas_td_binaural_renderer_unwrap() - * - * Call ivas_td_binaural_renderer() without st_ivas. - *---------------------------------------------------------------------*/ - -ivas_error ivas_td_binaural_renderer_unwrap( - const REVERB_HANDLE hReverb, /* i : Reverberator handle */ - const AUDIO_CONFIG transport_config, /* i : Transport configuration */ - BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD binaural object renderer handle */ - const int16_t num_src, /* i : number of sources to render */ - const int16_t lfe_idx, /* i : LFE channel index */ - const IVAS_FORMAT ivas_format, /* i : IVAS format */ - ISM_METADATA_HANDLE *hIsmMetaData, /* i : ISM metadata handle */ - const int16_t Opt_Headrotation, /* i : Head rotation flag */ - const IVAS_QUATERNION *Quaternions, /* i : Head tracking data per subframe */ - const IVAS_VECTOR3 *Pos, /* i : Listener position data per subframe */ -#ifdef JBM_TSM_ON_TCS - float *output[], /* i/o: SCE channels / Binaural synthesis */ -#else - float output[][L_FRAME48k], /* i/o: SCE channels / Binaural synthesis */ -#endif - const int16_t output_frame /* i : output frame length */ -) -{ - int16_t subframe_length; - int16_t subframe_idx; - float reverb_signal[BINAURAL_CHANNELS][L_FRAME48k]; - ivas_error error; -#ifdef JBM_TSM_ON_TCS - float *p_reverb_signal[BINAURAL_CHANNELS]; - int16_t ch; - - for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) - { - p_reverb_signal[ch] = reverb_signal[ch]; - } -#endif - - subframe_length = output_frame / MAX_PARAM_SPATIAL_SUBFRAMES; - -#ifdef FIX_356_ISM_METADATA_SYNC - int16_t c_indx, nS, subframe_update; - subframe_update = 0; - c_indx = 0; - - for ( nS = 0; nS < num_src; nS++ ) - { - if ( !( ivas_format == MC_FORMAT && nS == lfe_idx ) ) /* Skip LFE for MC */ - { - hBinRendererTd->Sources[c_indx]->InputFrame_p = output[nS]; - hBinRendererTd->Sources[c_indx]->SrcRend_p->InputAvailable = TRUE; - c_indx++; - } - } -#else - /* Update object position(s) */ - TDREND_Update_object_positions( hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, output ); -#endif - - for ( subframe_idx = 0; subframe_idx < MAX_PARAM_SPATIAL_SUBFRAMES; subframe_idx++ ) - { -#ifdef FIX_356_ISM_METADATA_SYNC - if ( subframe_idx == subframe_update ) - { - /* Update object position(s) */ - TDREND_Update_object_positions( hBinRendererTd, num_src, ivas_format, hIsmMetaData ); - } -#endif - /* Update the listener's location/orientation */ - TDREND_Update_listener_orientation( hBinRendererTd, Opt_Headrotation, ( Quaternions != NULL ) ? &Quaternions[subframe_idx] : NULL, ( Pos != NULL ) ? &Pos[subframe_idx] : NULL ); - - if ( hReverb != NULL && hReverb->pConfig.roomAcoustics.late_reverb_on ) - { -#ifdef JBM_TSM_ON_TCS - if ( ( error = ivas_reverb_process( hReverb, transport_config, 0, output, p_reverb_signal, subframe_idx ) ) != IVAS_ERR_OK ) -#else - if ( ( error = ivas_reverb_process( hReverb, transport_config, 0, output, reverb_signal, subframe_idx ) ) != IVAS_ERR_OK ) -#endif - { - return error; - } - } - - /* Render subframe */ -#ifdef FIX_356_ISM_METADATA_SYNC - if ( ( error = TDREND_GetMix( hBinRendererTd, output, subframe_length, subframe_idx, subframe_update ) ) != IVAS_ERR_OK ) -#else - if ( ( error = TDREND_GetMix( hBinRendererTd, output, subframe_length, subframe_idx ) ) != IVAS_ERR_OK ) -#endif - { - return error; - } - } - - - if ( hReverb != NULL && hReverb->pConfig.roomAcoustics.late_reverb_on ) - { - /* add reverb to rendered signals */ - v_add( reverb_signal[0], output[0], output[0], output_frame ); - v_add( reverb_signal[1], output[1], output[1], output_frame ); - } - - return IVAS_ERR_OK; -} - - -/*---------------------------------------------------------------------* - * TDREND_GetMix() - * - * Render one 5 ms subframe from the mixer - *---------------------------------------------------------------------*/ - -ivas_error TDREND_GetMix( - BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD renderer handle */ -#ifdef JBM_TSM_ON_TCS - float *output[], /* i/o: ISM object synth / rendered output in 0,1 */ -#else - float output[][L_FRAME48k], /* i/o: ISM object synth / rendered output in 0,1 */ -#endif - const int16_t subframe_length, /* i/o: subframe length */ - const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ -#ifdef FIX_356_ISM_METADATA_SYNC - , - const int16_t subframe_update -#endif -) -{ - int16_t i; - TDREND_SRC_t *Src_p; - TDREND_SRC_SPATIAL_t *SrcSpatial_p; - TDREND_SRC_REND_t *SrcRend_p; - ivas_error error; - float output_buf[2][L_SPATIAL_SUBFR_48k]; /* Temp buffer for left/right rendered signal */ - float hrf_left_delta[SFX_SPAT_BIN_MAX_FILTER_LENGTH]; - float hrf_right_delta[SFX_SPAT_BIN_MAX_FILTER_LENGTH]; - int16_t intp_count; -#ifdef ISM_NON_DIEGETIC_PAN - float pan_left, pan_right; -#endif -#ifdef FIX_356_ISM_METADATA_SYNC - int16_t subframe_update_flag; - subframe_update_flag = subframe_idx == subframe_update; -#endif - error = IVAS_ERR_OK; - - /* Clear the output buffer to accumulate rendered sources */ - set_f( output_buf[0], 0.0f, subframe_length ); - set_f( output_buf[1], 0.0f, subframe_length ); - - /* Clear interpolation buffers and counter */ - set_f( hrf_left_delta, 0.0f, SFX_SPAT_BIN_MAX_FILTER_LENGTH ); - set_f( hrf_right_delta, 0.0f, SFX_SPAT_BIN_MAX_FILTER_LENGTH ); - intp_count = 0; - - /* Create the mix */ - /* Loop through the source list and render each source */ - for ( i = 0; i < hBinRendererTd->NumOfSrcs; i++ ) - { - Src_p = hBinRendererTd->Sources[i]; - SrcSpatial_p = Src_p->SrcSpatial_p; - SrcRend_p = Src_p->SrcRend_p; - - /* Update rendering params if needed */ -#ifdef ISM_NON_DIEGETIC_PAN - if ( ( SrcRend_p->PlayStatus == TDREND_PLAYSTATUS_PLAYING ) && ( hBinRendererTd->Listener_p->PoseUpdated || SrcSpatial_p->Updated ) ) -#else - if ( hBinRendererTd->Listener_p->PoseUpdated || SrcSpatial_p->Updated ) -#endif - { - TDREND_SRC_REND_UpdateFiltersFromSpatialParams( hBinRendererTd, SrcRend_p, SrcSpatial_p, Src_p->hrf_left_prev, - Src_p->hrf_right_prev, hrf_left_delta, hrf_right_delta, &intp_count, &Src_p->filterlength, &Src_p->itd, &Src_p->Gain, Src_p, -#ifdef FIX_356_ISM_METADATA_SYNC - subframe_update_flag -#else - subframe_idx -#endif - ); - } - - /* Render source if needed */ - if ( ( SrcRend_p->InputAvailable == TRUE ) && ( SrcRend_p->PlayStatus == TDREND_PLAYSTATUS_PLAYING ) ) - { - error = TDREND_REND_RenderSourceHRFilt( Src_p, hrf_left_delta, hrf_right_delta, intp_count, output_buf, subframe_length ); - } - -#ifdef ISM_NON_DIEGETIC_PAN - if ( ( SrcRend_p->InputAvailable == TRUE ) && ( SrcRend_p->PlayStatus == TDREND_PLAYSTATUS_PLAYING_NON_DIEGETIC ) ) - { - pan_left = ( SrcSpatial_p->Pos_p[1] + 1.f ) * 0.5f; - pan_right = 1.f - pan_left; - v_multc_acc( &Src_p->InputFrame_p[subframe_idx * subframe_length], pan_left, output_buf[0], subframe_length ); - v_multc_acc( &Src_p->InputFrame_p[subframe_idx * subframe_length], pan_right, output_buf[1], subframe_length ); - } -#endif - } - - /* Populate output variable */ - mvr2r( output_buf[0], output[0] + subframe_idx * subframe_length, subframe_length ); /* Left */ - mvr2r( output_buf[1], output[1] + subframe_idx * subframe_length, subframe_length ); /* Right */ - - /* Clear the PoseUpdated and Source position update flags */ - TDREND_Clear_Update_flags( hBinRendererTd ); - - return error; -} - - -/*---------------------------------------------------------------------* - * TDREND_Clear_Update_flags() - * - * Initializes the audio mixer module - *---------------------------------------------------------------------*/ - -static void TDREND_Clear_Update_flags( - BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd /* i/o: TD renderer handle */ -) -{ - int16_t i; - - hBinRendererTd->Listener_p->PoseUpdated = FALSE; - - for ( i = 0; i < hBinRendererTd->NumOfSrcs; i++ ) - { - hBinRendererTd->Sources[i]->SrcSpatial_p->Updated = FALSE; - } - - return; -} - - -/*---------------------------------------------------------------------* - * TDREND_Update_object_positions() - * - * Update object position(s) - *---------------------------------------------------------------------*/ - -void TDREND_Update_object_positions( - BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o : TD Renderer handle */ - const int16_t num_src, /* i : number of sources to render */ -#ifndef FIX_356_ISM_METADATA_SYNC - const int16_t lfe_idx, /* i : Input LFE index */ -#endif - const IVAS_FORMAT in_format, /* i : Format of input sources */ - const ISM_METADATA_HANDLE *hIsmMetaData /* i : Input metadata for ISM objects */ -#ifndef FIX_356_ISM_METADATA_SYNC - , -#ifdef JBM_TSM_ON_TCS - float *output[] -#else - float output[][L_FRAME48k] /* i/o: SCE/MC channels */ -#endif -#endif -) -{ - TDREND_DirAtten_t *DirAtten_p; - int16_t nS; - float Pos[3]; - float Dir[3]; -#ifndef FIX_356_ISM_METADATA_SYNC - int16_t c_indx; -#endif - - DirAtten_p = hBinRendererTd->DirAtten_p; - - /* For each source, write the frame data to the source object*/ -#ifndef FIX_356_ISM_METADATA_SYNC - c_indx = 0; -#endif - for ( nS = 0; nS < num_src; nS++ ) - { -#ifndef FIX_356_ISM_METADATA_SYNC - if ( !( in_format == MC_FORMAT && nS == lfe_idx ) ) /* Skip LFE for MC */ - { - hBinRendererTd->Sources[c_indx]->InputFrame_p = output[nS]; - hBinRendererTd->Sources[c_indx]->SrcRend_p->InputAvailable = TRUE; - c_indx++; - } -#endif - if ( in_format == ISM_FORMAT ) - { - /* Update the source positions */ - /* Source position and direction */ - angles_to_vec( hIsmMetaData[nS]->radius, hIsmMetaData[nS]->azimuth, hIsmMetaData[nS]->elevation, Pos ); - angles_to_vec( 1.0f, hIsmMetaData[nS]->yaw, hIsmMetaData[nS]->pitch, Dir ); - -#ifndef FIX_463_TD_RENDERER_DIRECTIVITY_RESET - - /* Source directivity info */ - DirAtten_p->ConeInnerAngle = 360.0f; - DirAtten_p->ConeOuterAngle = 360.0f; - DirAtten_p->ConeOuterGain = 1.0f; -#endif - - TDREND_MIX_SRC_SetPos( hBinRendererTd, nS, Pos ); - TDREND_MIX_SRC_SetDirAtten( hBinRendererTd, nS, DirAtten_p ); - -#ifdef ISM_NON_DIEGETIC_PAN - if ( hIsmMetaData[nS]->non_diegetic_flag ) - { - Pos[0] = 0; - Pos[1] = hIsmMetaData[nS]->azimuth / 90.f; - Pos[2] = 0; - TDREND_MIX_SRC_SetPos( hBinRendererTd, nS, Pos ); - TDREND_MIX_SRC_SetPlayState( hBinRendererTd, nS, TDREND_PLAYSTATUS_PLAYING_NON_DIEGETIC ); - } - else - { - TDREND_MIX_SRC_SetPlayState( hBinRendererTd, nS, TDREND_PLAYSTATUS_PLAYING ); - } -#else - TDREND_MIX_SRC_SetPlayState( hBinRendererTd, nS, TDREND_PLAYSTATUS_PLAYING ); -#endif - TDREND_MIX_SRC_SetDir( hBinRendererTd, nS, Dir ); - } - } - - return; -} - - -/*---------------------------------------------------------------------* - * TDREND_Update_listener_orientation() - * - * Update listener orientation (s) - *---------------------------------------------------------------------*/ - -void TDREND_Update_listener_orientation( - BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD Renderer handle */ - const int16_t headRotEnabled, /* i : Headrotation flag */ - const IVAS_QUATERNION *headPosition, /* i : Listener orientation */ - const IVAS_VECTOR3 *Pos /* i : Listener Position */ -) -{ - float FrontVec[3]; - float UpVec[3]; - float Rmat[3][3]; - float Pos_p[3]; - - if ( headRotEnabled ) - { - /* Obtain head rotation matrix */ - QuatToRotMat( *headPosition, Rmat ); - /* Apply rotation matrix to looking vector [1;0;0] */ - FrontVec[0] = Rmat[0][0]; - FrontVec[1] = Rmat[0][1]; - FrontVec[2] = Rmat[0][2]; - /* Apply rotation matrix to up vector [0;0;1] */ - UpVec[0] = Rmat[2][0]; - UpVec[1] = Rmat[2][1]; - UpVec[2] = Rmat[2][2]; - /* Input position */ - Pos_p[0] = ( *Pos ).x; - Pos_p[1] = ( *Pos ).y; - Pos_p[2] = ( *Pos ).z; - } - else - { - /* Oriented with looking vector along the x axis */ - FrontVec[0] = 1.0f; - FrontVec[1] = 0.0f; - FrontVec[2] = 0.0f; - /* Oriented with up vector along the z axis */ - UpVec[0] = 0.0f; - UpVec[1] = 0.0f; - UpVec[2] = 1.0f; - /* Listener at the origin */ - Pos_p[0] = 0.0f; - Pos_p[1] = 0.0f; - Pos_p[2] = 0.0f; - } - - /* Set the listener position and orientation:*/ - TDREND_MIX_LIST_SetPos( hBinRendererTd, Pos_p ); - TDREND_MIX_LIST_SetOrient( hBinRendererTd, FrontVec, UpVec ); - - return; -} - - -/*---------------------------------------------------------------------* - * ivas_td_binaural_open_ext() - * - * - *---------------------------------------------------------------------*/ - -ivas_error ivas_td_binaural_open_ext( - TDREND_WRAPPER *pTDRend, - IVAS_REND_AudioConfig inConfig, - RENDER_CONFIG_DATA *hRendCfg, /* i : Renderer configuration */ - LSSETUP_CUSTOM_STRUCT *customLsInput, - const int32_t outFs ) -{ - int16_t nchan_transport; - AUDIO_CONFIG transport_config; - IVAS_FORMAT ivas_format; - IVAS_OUTPUT_SETUP hTransSetup; - ivas_error error; - float *directivity = NULL; - - if ( inConfig != IVAS_REND_AUDIO_CONFIG_LS_CUSTOM ) - { - if ( ( error = getAudioConfigNumChannels( inConfig, &nchan_transport ) ) != IVAS_ERR_OK ) - { - return error; - } - } - else - { - nchan_transport = customLsInput->num_spk; - } - - transport_config = getIvasAudioConfigFromRendAudioConfig( inConfig ); - ivas_format = ( getAudioConfigType( inConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) ? MC_FORMAT : ISM_FORMAT; - hTransSetup.ls_azimuth = customLsInput->ls_azimuth; - hTransSetup.ls_elevation = customLsInput->ls_elevation; - - if ( NULL != hRendCfg ) - { - directivity = hRendCfg->directivity; - } - - return ivas_td_binaural_open_unwrap( &pTDRend->hHrtfTD, outFs, nchan_transport, ivas_format, transport_config, directivity, hTransSetup, &pTDRend->hBinRendererTd, &pTDRend->binaural_latency_ns ); -} - - -/*---------------------------------------------------------------------* - * ivas_td_binaural_renderer_ext() - * - * Receives the current frames for the object streams, updates metadata - * and renders the current frame. - *---------------------------------------------------------------------*/ - -ivas_error ivas_td_binaural_renderer_ext( - const TDREND_WRAPPER *pTDRend, /* i : TD Renderer wrapper structure */ - const IVAS_REND_AudioConfig inConfig, /* i : Input audio configuration */ - const LSSETUP_CUSTOM_STRUCT *customLsInput, /* i : Input custom loudspeaker layout */ - const IVAS_REND_HeadRotData *headRotData, /* i : Input head positions */ - const IVAS_REND_AudioObjectPosition *currentPos, /* i : Object position */ - const REVERB_HANDLE hReverb, /* i : Reverberator handle */ - const int16_t output_frame, /* i : output frame length */ - float output[][L_FRAME48k] /* i/o: SCE channels / Binaural synthesis */ -) -{ - ISM_METADATA_FRAME hIsmMetaDataFrame; - ISM_METADATA_HANDLE hIsmMetaData[1]; - int16_t lfe_idx; - int16_t num_src; - IVAS_FORMAT ivas_format; - IVAS_REND_AudioConfigType inConfigType; - AUDIO_CONFIG transport_config; - ivas_error error; -#ifdef JBM_TSM_ON_TCS - float *p_output[MAX_OUTPUT_CHANNELS]; - int16_t ch; - - for ( ch = 0; ch < MAX_OUTPUT_CHANNELS; ch++ ) - { - p_output[ch] = output[ch]; - } -#endif - - push_wmops( "ivas_td_binaural_renderer_ext" ); - - inConfigType = getAudioConfigType( inConfig ); - lfe_idx = LFE_CHANNEL; - hIsmMetaData[0] = NULL; - - if ( inConfigType == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) - { - ivas_format = MC_FORMAT; - transport_config = getIvasAudioConfigFromRendAudioConfig( inConfig ); - if ( inConfig != IVAS_REND_AUDIO_CONFIG_LS_CUSTOM ) - { - if ( ( error = getAudioConfigNumChannels( inConfig, &num_src ) ) != IVAS_ERR_OK ) - { - return error; - } - } - else - { - lfe_idx = ( customLsInput->num_lfe > 0 ) ? customLsInput->lfe_idx[0] : -1; - num_src = customLsInput->num_spk + customLsInput->num_lfe; - } - } - else - { - ivas_format = ISM_FORMAT; - num_src = 1; - transport_config = AUDIO_CONFIG_ISM1; - hIsmMetaData[0] = &hIsmMetaDataFrame; - hIsmMetaData[0]->azimuth = currentPos->azimuth; - hIsmMetaData[0]->elevation = currentPos->elevation; - hIsmMetaData[0]->yaw = currentPos->yaw; - hIsmMetaData[0]->pitch = currentPos->pitch; - hIsmMetaData[0]->radius = currentPos->radius; -#ifdef ISM_NON_DIEGETIC_PAN - hIsmMetaData[0]->non_diegetic_flag = currentPos->non_diegetic_flag; -#endif - } - -#ifdef JBM_TSM_ON_TCS - if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, - ( headRotData != NULL ) ? headRotData->headPositions : NULL, - ( headRotData != NULL ) ? headRotData->Pos : NULL, p_output, output_frame ) ) != IVAS_ERR_OK ) -#else - if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, - ( headRotData != NULL ) ? headRotData->headPositions : NULL, - ( headRotData != NULL ) ? headRotData->Pos : NULL, output, output_frame ) ) != IVAS_ERR_OK ) - -#endif - { - return error; - } - - pop_wmops(); - - return IVAS_ERR_OK; -} - -/*---------------------------------------------------------------------* - * angles_to_vec() - * - * Convert azimuth and elevation angles to position/orientation vector - *---------------------------------------------------------------------*/ - -static void angles_to_vec( - const float radius, /* i : radius */ - const float azimuth, /* i : Azimuth angle */ - const float elevation, /* i : Elevation angle */ - float *vec /* o : Pos/Dir vector */ -) -{ - vec[0] = radius * cosf( elevation * PI_OVER_180 ) * cosf( azimuth * PI_OVER_180 ); - vec[1] = radius * cosf( elevation * PI_OVER_180 ) * sinf( azimuth * PI_OVER_180 ); - vec[2] = radius * sinf( elevation * PI_OVER_180 ); - - return; -} -- GitLab From 6718d738d46192a082b98d858c535ed1ba37e909 Mon Sep 17 00:00:00 2001 From: Sumeyra Kanik Date: Wed, 17 May 2023 17:20:29 +0200 Subject: [PATCH 11/18] =?UTF-8?q?Delay=20=C3=ADsm=20metadata=20only=20for?= =?UTF-8?q?=20decoder/renderer,=20not=20for=20Ivas=5Frend?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib_dec/ivas_objectRenderer_internal.c | 11 +++++++++- lib_rend/ivas_objectRenderer.c | 30 ++++++++++++++++++++------ lib_rend/ivas_prot_rend.h | 3 +++ 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/lib_dec/ivas_objectRenderer_internal.c b/lib_dec/ivas_objectRenderer_internal.c index 47f950a550..0867bf22ce 100644 --- a/lib_dec/ivas_objectRenderer_internal.c +++ b/lib_dec/ivas_objectRenderer_internal.c @@ -75,12 +75,21 @@ ivas_error ivas_td_binaural_renderer( const int16_t output_frame /* i : output frame length */ ) { +#ifdef FIX_356_ISM_METADATA_SYNC + int16_t ism_md_subframe_update = 2; return ivas_td_binaural_renderer_unwrap( + st_ivas->hReverb, + st_ivas->transport_config, + st_ivas->hBinRendererTd, st_ivas->nchan_transport, LFE_CHANNEL, st_ivas->ivas_format, + st_ivas->hIsmMetaData, st_ivas->hDecoderConfig->Opt_Headrotation, (st_ivas->hHeadTrackData != NULL) ? st_ivas->hHeadTrackData->Quaternions : NULL, + (st_ivas->hHeadTrackData != NULL) ? st_ivas->hHeadTrackData->Pos : NULL, ism_md_subframe_update, output, output_frame); +#else return ivas_td_binaural_renderer_unwrap( st_ivas->hReverb, st_ivas->transport_config, st_ivas->hBinRendererTd, st_ivas->nchan_transport, LFE_CHANNEL, st_ivas->ivas_format, st_ivas->hIsmMetaData, st_ivas->hDecoderConfig->Opt_Headrotation, ( st_ivas->hHeadTrackData != NULL ) ? st_ivas->hHeadTrackData->Quaternions : NULL, ( st_ivas->hHeadTrackData != NULL ) ? st_ivas->hHeadTrackData->Pos : NULL, output, output_frame ); +#endif } #ifdef JBM_TSM_ON_TCS @@ -103,7 +112,7 @@ void ObjRenderIVASSubframe( float *tc_local[MAX_TRANSPORT_CHANNELS]; int16_t ch, slot_size, slots_to_render, output_frame; #ifdef FIX_356_ISM_METADATA_SYNC - int16_t subframe_update = 0; + int16_t subframe_update = 0; /* Number of subframes to delay metadata to sync with audio */ #endif for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) { diff --git a/lib_rend/ivas_objectRenderer.c b/lib_rend/ivas_objectRenderer.c index 36e187870f..3db591d094 100644 --- a/lib_rend/ivas_objectRenderer.c +++ b/lib_rend/ivas_objectRenderer.c @@ -265,6 +265,9 @@ ivas_error ivas_td_binaural_renderer_unwrap( const int16_t Opt_Headrotation, /* i : Head rotation flag */ const IVAS_QUATERNION *Quaternions, /* i : Head tracking data per subframe */ const IVAS_VECTOR3 *Pos, /* i : Listener position data per subframe */ +#ifdef FIX_356_ISM_METADATA_SYNC + const int16_t ism_md_subframe_update, /* i: Number of subframes to delay ism metadata to sync with audio */ +#endif #ifdef JBM_TSM_ON_TCS float *output[], /* i/o: SCE channels / Binaural synthesis */ #else @@ -290,8 +293,7 @@ ivas_error ivas_td_binaural_renderer_unwrap( subframe_length = output_frame / MAX_PARAM_SPATIAL_SUBFRAMES; #ifdef FIX_356_ISM_METADATA_SYNC - int16_t c_indx, nS, subframe_update; - subframe_update = 0; + int16_t c_indx, nS; c_indx = 0; for ( nS = 0; nS < num_src; nS++ ) @@ -311,7 +313,7 @@ ivas_error ivas_td_binaural_renderer_unwrap( for ( subframe_idx = 0; subframe_idx < MAX_PARAM_SPATIAL_SUBFRAMES; subframe_idx++ ) { #ifdef FIX_356_ISM_METADATA_SYNC - if ( subframe_idx == subframe_update ) + if ( subframe_idx == ism_md_subframe_update ) { /* Update object position(s) */ TDREND_Update_object_positions( hBinRendererTd, num_src, ivas_format, hIsmMetaData ); @@ -334,7 +336,7 @@ ivas_error ivas_td_binaural_renderer_unwrap( /* Render subframe */ #ifdef FIX_356_ISM_METADATA_SYNC - if ( ( error = TDREND_GetMix( hBinRendererTd, output, subframe_length, subframe_idx, subframe_update ) ) != IVAS_ERR_OK ) + if ( ( error = TDREND_GetMix( hBinRendererTd, output, subframe_length, subframe_idx, ism_md_subframe_update ) ) != IVAS_ERR_OK ) #else if ( ( error = TDREND_GetMix( hBinRendererTd, output, subframe_length, subframe_idx ) ) != IVAS_ERR_OK ) #endif @@ -372,7 +374,7 @@ ivas_error TDREND_GetMix( const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ #ifdef FIX_356_ISM_METADATA_SYNC , - const int16_t subframe_update + const int16_t ism_md_subframe_update /* Number of subframes to delay ism metadata to sync with audio */ #endif ) { @@ -390,7 +392,7 @@ ivas_error TDREND_GetMix( #endif #ifdef FIX_356_ISM_METADATA_SYNC int16_t subframe_update_flag; - subframe_update_flag = subframe_idx == subframe_update; + subframe_update_flag = subframe_idx == ism_md_subframe_update; #endif error = IVAS_ERR_OK; @@ -700,6 +702,9 @@ ivas_error ivas_td_binaural_renderer_ext( IVAS_REND_AudioConfigType inConfigType; AUDIO_CONFIG transport_config; ivas_error error; +#ifdef FIX_356_ISM_METADATA_SYNC + int16_t ism_md_subframe_update = 0; +#endif #ifdef JBM_TSM_ON_TCS float *p_output[MAX_OUTPUT_CHANNELS]; int16_t ch; @@ -750,14 +755,25 @@ ivas_error ivas_td_binaural_renderer_ext( } #ifdef JBM_TSM_ON_TCS +#ifdef FIX_356_ISM_METADATA_SYNC + if( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, + ( headRotData != NULL ) ? headRotData->headPositions : NULL, + ( headRotData != NULL ) ? headRotData->Pos : NULL, ism_md_subframe_update, p_output, output_frame ) ) != IVAS_ERR_OK ) +#else if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, ( headRotData != NULL ) ? headRotData->headPositions : NULL, ( headRotData != NULL ) ? headRotData->Pos : NULL, p_output, output_frame ) ) != IVAS_ERR_OK ) +#endif +#else +#ifdef FIX_356_ISM_METADATA_SYNC + if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, + ( headRotData != NULL ) ? headRotData->headPositions : NULL, + ( headRotData != NULL ) ? headRotData->Pos : NULL, ism_md_subframe_update, output, output_frame ) ) != IVAS_ERR_OK ) #else if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, ( headRotData != NULL ) ? headRotData->headPositions : NULL, ( headRotData != NULL ) ? headRotData->Pos : NULL, output, output_frame ) ) != IVAS_ERR_OK ) - +#endif #endif { return error; diff --git a/lib_rend/ivas_prot_rend.h b/lib_rend/ivas_prot_rend.h index 6a4f8fb125..9de3ca2345 100644 --- a/lib_rend/ivas_prot_rend.h +++ b/lib_rend/ivas_prot_rend.h @@ -246,6 +246,9 @@ ivas_error ivas_td_binaural_renderer_unwrap( const int16_t Opt_Headrotation, /* i : Head rotation flag */ const IVAS_QUATERNION *Quaternions, /* i : Head tracking data per subframe */ const IVAS_VECTOR3 *Pos, /* i : Listener position data per subframe */ +#ifdef FIX_356_ISM_METADATA_SYNC + const int16_t ism_md_subframe_update, +#endif #ifdef JBM_TSM_ON_TCS float *output[], /* i/o: SCE channels / Binaural synthesis */ #else -- GitLab From fce08feb09356dccdd40b4433eb07b460f45e5b5 Mon Sep 17 00:00:00 2001 From: Sumeyra Kanik Date: Wed, 17 May 2023 17:26:44 +0200 Subject: [PATCH 12/18] Format fix --- lib_dec/ivas_objectRenderer_internal.c | 11 ++++++----- lib_rend/ivas_prot_rend.h | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lib_dec/ivas_objectRenderer_internal.c b/lib_dec/ivas_objectRenderer_internal.c index 0867bf22ce..3f7d05a07a 100644 --- a/lib_dec/ivas_objectRenderer_internal.c +++ b/lib_dec/ivas_objectRenderer_internal.c @@ -81,9 +81,10 @@ ivas_error ivas_td_binaural_renderer( st_ivas->hReverb, st_ivas->transport_config, st_ivas->hBinRendererTd, st_ivas->nchan_transport, LFE_CHANNEL, st_ivas->ivas_format, - st_ivas->hIsmMetaData, st_ivas->hDecoderConfig->Opt_Headrotation, (st_ivas->hHeadTrackData != NULL) ? st_ivas->hHeadTrackData->Quaternions : NULL, - (st_ivas->hHeadTrackData != NULL) ? st_ivas->hHeadTrackData->Pos : NULL, ism_md_subframe_update, output, output_frame); -#else return ivas_td_binaural_renderer_unwrap( + st_ivas->hIsmMetaData, st_ivas->hDecoderConfig->Opt_Headrotation, ( st_ivas->hHeadTrackData != NULL ) ? st_ivas->hHeadTrackData->Quaternions : NULL, + ( st_ivas->hHeadTrackData != NULL ) ? st_ivas->hHeadTrackData->Pos : NULL, ism_md_subframe_update, output, output_frame ); +#else + return ivas_td_binaural_renderer_unwrap( st_ivas->hReverb, st_ivas->transport_config, st_ivas->hBinRendererTd, st_ivas->nchan_transport, LFE_CHANNEL, st_ivas->ivas_format, @@ -112,7 +113,7 @@ void ObjRenderIVASSubframe( float *tc_local[MAX_TRANSPORT_CHANNELS]; int16_t ch, slot_size, slots_to_render, output_frame; #ifdef FIX_356_ISM_METADATA_SYNC - int16_t subframe_update = 0; /* Number of subframes to delay metadata to sync with audio */ + int16_t ism_md_subframe_update = 0; /* Number of subframes to delay metadata to sync with audio */ #endif for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) { @@ -162,7 +163,7 @@ void ObjRenderIVASSubframe( /* Render subframe */ #ifdef FIX_356_ISM_METADATA_SYNC - TDREND_GetMix( st_ivas->hBinRendererTd, output_f_local, output_frame, 0, subframe_update ); + TDREND_GetMix( st_ivas->hBinRendererTd, output_f_local, output_frame, 0, ism_md_subframe_update ); #else TDREND_GetMix( st_ivas->hBinRendererTd, output_f_local, output_frame, 0 ); #endif diff --git a/lib_rend/ivas_prot_rend.h b/lib_rend/ivas_prot_rend.h index 9de3ca2345..03131430ed 100644 --- a/lib_rend/ivas_prot_rend.h +++ b/lib_rend/ivas_prot_rend.h @@ -304,7 +304,7 @@ ivas_error TDREND_GetMix( const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ #ifdef FIX_356_ISM_METADATA_SYNC , - const int16_t subframe_update /* Number of subframes to delay metadata to sync with audio */ + const int16_t ism_md_subframe_update /* Number of subframes to delay metadata to sync with audio */ #endif ); -- GitLab From 27d36320b54c6fb2d5359c352f9149c17c86f1b9 Mon Sep 17 00:00:00 2001 From: Sumeyra Kanik Date: Wed, 17 May 2023 17:39:32 +0200 Subject: [PATCH 13/18] Format fix --- lib_rend/ivas_objectRenderer.c | 805 +++++++++++++++++++++++++++++++++ 1 file changed, 805 insertions(+) diff --git a/lib_rend/ivas_objectRenderer.c b/lib_rend/ivas_objectRenderer.c index 3db591d094..0eec81b692 100644 --- a/lib_rend/ivas_objectRenderer.c +++ b/lib_rend/ivas_objectRenderer.c @@ -803,3 +803,808 @@ static void angles_to_vec( return; } +/****************************************************************************************************** + + (C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository. All Rights Reserved. + + This software is protected by copyright law and by international treaties. + The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository retain full ownership rights in their respective contributions in + the software. This notice grants no license of any kind, including but not limited to patent + license, nor is any license granted by implication, estoppel or otherwise. + + Contributors are required to enter into the IVAS codec Public Collaboration agreement before making + contributions. + + This software is provided "AS IS", without any express or implied warranties. The software is in the + development stage. It is intended exclusively for experts who have experience with such software and + solely for the purpose of inspection. All implied warranties of non-infringement, merchantability + and fitness for a particular purpose are hereby disclaimed and excluded. + + Any dispute, controversy or claim arising under or in relation to providing this software shall be + submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in + accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and + the United Nations Convention on Contracts on the International Sales of Goods. + +*******************************************************************************************************/ + +#include "ivas_stat_rend.h" +#include +#include "options.h" +#include "prot.h" +#ifdef ISM_NON_DIEGETIC_PAN +#include "ivas_prot.h" +#endif +#include "ivas_prot_rend.h" +#include +#include "ivas_rom_com.h" +#ifdef DEBUGGING +#include "debug.h" +#endif +#include "wmc_auto.h" + + +/*---------------------------------------------------------------------* + * Local function prototypes + *---------------------------------------------------------------------*/ + + +static void TDREND_Clear_Update_flags( BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd ); + +static void angles_to_vec( const float radius, const float azimuth, const float elevation, float *vec ); + + +/*---------------------------------------------------------------------* + * ivas_td_binaural_open_unwrap() + * + * Call TD open/init function without st_ivas + *---------------------------------------------------------------------*/ + +ivas_error ivas_td_binaural_open_unwrap( + TDREND_HRFILT_FiltSet_t **hHrtfTD, /* i/o: HR filter model (from file or NULL) */ + const int32_t output_Fs, /* i : Output sampling rate */ + const int16_t nchan_transport, /* i : Number of channels */ + const IVAS_FORMAT ivas_format, /* i : IVAS format (ISM/MC) */ + const AUDIO_CONFIG transport_config, /* i : Transport configuration */ + const float *directivity, /* i : Directivity pattern (used for ISM) */ + const IVAS_OUTPUT_SETUP hTransSetup, /* i : Loudspeaker layout */ + BINAURAL_TD_OBJECT_RENDERER_HANDLE *hBinRendererTd, /* o : TD renderer handle */ + int32_t *binaural_latency_ns /* i : Binauralization delay */ +) +{ + BINAURAL_TD_OBJECT_RENDERER_HANDLE pBinRendTd; + TDREND_PosType_t PosType; + int16_t nS; + int16_t SrcInd[MAX_NUM_TDREND_CHANNELS]; + const float *ls_azimuth, *ls_elevation; + float Pos[3]; + float Dir[3]; + TDREND_DirAtten_t *DirAtten_p; + int16_t nchan_rend; + ivas_error error; + + error = IVAS_ERR_OK; + + if ( ( pBinRendTd = malloc( sizeof( BINAURAL_TD_OBJECT_RENDERER ) ) ) == NULL ) + { + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for TD renderer\n" ) ); + } + if ( ( pBinRendTd->TdRend_MixSpatSpec_p = malloc( sizeof( TDREND_MixSpatSpec_t ) ) ) == NULL ) + { + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for TD renderer\n" ) ); + } + if ( ( pBinRendTd->DirAtten_p = malloc( sizeof( TDREND_DirAtten_t ) ) ) == NULL ) + { + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for TD renderer\n" ) ); + } + if ( ( pBinRendTd->Listener_p = malloc( sizeof( TDREND_MIX_Listener_t ) ) ) == NULL ) + { + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for TD renderer\n" ) ); + } + + pBinRendTd->NumOfSrcs = 0; + pBinRendTd->MaxSrcInd = -1; + + /* Mixer spatial setup */ + pBinRendTd->TdRend_MixSpatSpec_p->UseCommonDistAttenModel = TRUE; + pBinRendTd->TdRend_MixSpatSpec_p->DistAttenModel = 0; /* 0=Turned off, else use TDREND_DIST_ATTEN_MODEL_INV_DIST_CLAMPED */ + + if ( ( error = TDREND_MIX_Init( pBinRendTd, hHrtfTD, pBinRendTd->TdRend_MixSpatSpec_p, output_Fs ) ) != IVAS_ERR_OK ) + { + return error; + } + + /* Set the attenuation (or can set MixSpatSpec.DistAttenModel above) */ + if ( ( error = TDREND_MIX_SetDistAttenModel( pBinRendTd, TDREND_DIST_ATTEN_MODEL_INV_DIST_CLAMPED ) ) != IVAS_ERR_OK ) + { + return error; + } + + /* Add sources to module and mixer, headphones */ + PosType = TDREND_POSTYPE_ABSOLUTE; /* or TDREND_POSTYPE_RELATIVE_TO_LISTENER */ + + nchan_rend = nchan_transport; + if ( ( ivas_format == MC_FORMAT ) && ( transport_config != AUDIO_CONFIG_LS_CUSTOM ) ) + { + nchan_rend--; /* Skip LFE channel -- added to the others */ + } + + for ( nS = 0; nS < nchan_rend; nS++ ) + { + if ( ( error = TDREND_MIX_AddSrc( pBinRendTd, &SrcInd[nS], PosType ) ) != IVAS_ERR_OK ) + { + return error; + } + } + + if ( ivas_format == MC_FORMAT ) + { + switch ( transport_config ) + { + case AUDIO_CONFIG_5_1: + ls_azimuth = ls_azimuth_CICP6; + ls_elevation = ls_elevation_CICP6; + break; + case AUDIO_CONFIG_7_1: + ls_azimuth = ls_azimuth_CICP12; + ls_elevation = ls_elevation_CICP12; + break; + case AUDIO_CONFIG_5_1_2: + ls_azimuth = ls_azimuth_CICP14; + ls_elevation = ls_elevation_CICP14; + break; + case AUDIO_CONFIG_5_1_4: + ls_azimuth = ls_azimuth_CICP16; + ls_elevation = ls_elevation_CICP16; + break; + case AUDIO_CONFIG_7_1_4: + ls_azimuth = ls_azimuth_CICP19; + ls_elevation = ls_elevation_CICP19; + break; + case AUDIO_CONFIG_LS_CUSTOM: + ls_azimuth = hTransSetup.ls_azimuth; + ls_elevation = hTransSetup.ls_elevation; + break; + default: + ls_azimuth = NULL; + ls_elevation = NULL; + } + + DirAtten_p = pBinRendTd->DirAtten_p; + + for ( nS = 0; nS < nchan_rend; nS++ ) + { + /* Set source positions according to loudspeaker layout */ + angles_to_vec( 1.0f, ls_azimuth[nS], ls_elevation[nS], Pos ); + Dir[0] = 1.0f; + Dir[1] = 0.0f; + Dir[2] = 0.0f; + + /* Source directivity info */ + DirAtten_p->ConeInnerAngle = 360.0f; + DirAtten_p->ConeOuterAngle = 360.0f; + DirAtten_p->ConeOuterGain = 1.0f; + + TDREND_MIX_SRC_SetPos( pBinRendTd, nS, Pos ); + TDREND_MIX_SRC_SetDir( pBinRendTd, nS, Dir ); + TDREND_MIX_SRC_SetPlayState( pBinRendTd, nS, TDREND_PLAYSTATUS_PLAYING ); + TDREND_MIX_SRC_SetDirAtten( pBinRendTd, nS, DirAtten_p ); + } + } + if ( ivas_format == ISM_FORMAT ) + { + DirAtten_p = pBinRendTd->DirAtten_p; + if ( NULL == directivity ) + { + DirAtten_p->ConeInnerAngle = 360.0f; /* Front cone */ + DirAtten_p->ConeOuterAngle = 360.0f; /* Back cone */ + DirAtten_p->ConeOuterGain = 1.0f; /* Back attenuation */ + } + else + { + DirAtten_p->ConeInnerAngle = directivity[0]; + DirAtten_p->ConeOuterAngle = directivity[1]; + DirAtten_p->ConeOuterGain = directivity[2]; + } + + for ( nS = 0; nS < nchan_rend; nS++ ) + { + TDREND_MIX_SRC_SetDirAtten( pBinRendTd, nS, DirAtten_p ); + } + } + + *hBinRendererTd = pBinRendTd; + *binaural_latency_ns = (int32_t) ( ( *hBinRendererTd )->HrFiltSet_p->latency_s * 1000000000.f ); + + return error; +} + + +/*---------------------------------------------------------------------* + * ivas_td_binaural_close() + * + * Close TD Object binaural renderer + *---------------------------------------------------------------------*/ + +void ivas_td_binaural_close( + BINAURAL_TD_OBJECT_RENDERER_HANDLE *hBinRendererTd /* i/o: TD binaural object renderer handle */ +) +{ + if ( hBinRendererTd == NULL || *hBinRendererTd == NULL ) + { + return; + } + + free( ( *hBinRendererTd )->TdRend_MixSpatSpec_p ); + free( ( *hBinRendererTd )->DirAtten_p ); + + TDREND_MIX_Dealloc( *hBinRendererTd ); + + free( *hBinRendererTd ); + *hBinRendererTd = NULL; + + return; +} + +/*---------------------------------------------------------------------* + * ivas_td_binaural_renderer_unwrap() + * + * Call ivas_td_binaural_renderer() without st_ivas. + *---------------------------------------------------------------------*/ + +ivas_error ivas_td_binaural_renderer_unwrap( + const REVERB_HANDLE hReverb, /* i : Reverberator handle */ + const AUDIO_CONFIG transport_config, /* i : Transport configuration */ + BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD binaural object renderer handle */ + const int16_t num_src, /* i : number of sources to render */ + const int16_t lfe_idx, /* i : LFE channel index */ + const IVAS_FORMAT ivas_format, /* i : IVAS format */ + ISM_METADATA_HANDLE *hIsmMetaData, /* i : ISM metadata handle */ + const int16_t Opt_Headrotation, /* i : Head rotation flag */ + const IVAS_QUATERNION *Quaternions, /* i : Head tracking data per subframe */ + const IVAS_VECTOR3 *Pos, /* i : Listener position data per subframe */ +#ifdef FIX_356_ISM_METADATA_SYNC + const int16_t ism_md_subframe_update, /* i: Number of subframes to delay ism metadata to sync with audio */ +#endif +#ifdef JBM_TSM_ON_TCS + float *output[], /* i/o: SCE channels / Binaural synthesis */ +#else + float output[][L_FRAME48k], /* i/o: SCE channels / Binaural synthesis */ +#endif + const int16_t output_frame /* i : output frame length */ +) +{ + int16_t subframe_length; + int16_t subframe_idx; + float reverb_signal[BINAURAL_CHANNELS][L_FRAME48k]; + ivas_error error; +#ifdef JBM_TSM_ON_TCS + float *p_reverb_signal[BINAURAL_CHANNELS]; + int16_t ch; + + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + p_reverb_signal[ch] = reverb_signal[ch]; + } +#endif + + subframe_length = output_frame / MAX_PARAM_SPATIAL_SUBFRAMES; + +#ifdef FIX_356_ISM_METADATA_SYNC + int16_t c_indx, nS; + c_indx = 0; + + for ( nS = 0; nS < num_src; nS++ ) + { + if ( !( ivas_format == MC_FORMAT && nS == lfe_idx ) ) /* Skip LFE for MC */ + { + hBinRendererTd->Sources[c_indx]->InputFrame_p = output[nS]; + hBinRendererTd->Sources[c_indx]->SrcRend_p->InputAvailable = TRUE; + c_indx++; + } + } +#else + /* Update object position(s) */ + TDREND_Update_object_positions( hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, output ); +#endif + + for ( subframe_idx = 0; subframe_idx < MAX_PARAM_SPATIAL_SUBFRAMES; subframe_idx++ ) + { +#ifdef FIX_356_ISM_METADATA_SYNC + if ( subframe_idx == ism_md_subframe_update ) + { + /* Update object position(s) */ + TDREND_Update_object_positions( hBinRendererTd, num_src, ivas_format, hIsmMetaData ); + } +#endif + /* Update the listener's location/orientation */ + TDREND_Update_listener_orientation( hBinRendererTd, Opt_Headrotation, ( Quaternions != NULL ) ? &Quaternions[subframe_idx] : NULL, ( Pos != NULL ) ? &Pos[subframe_idx] : NULL ); + + if ( hReverb != NULL && hReverb->pConfig.roomAcoustics.late_reverb_on ) + { +#ifdef JBM_TSM_ON_TCS + if ( ( error = ivas_reverb_process( hReverb, transport_config, 0, output, p_reverb_signal, subframe_idx ) ) != IVAS_ERR_OK ) +#else + if ( ( error = ivas_reverb_process( hReverb, transport_config, 0, output, reverb_signal, subframe_idx ) ) != IVAS_ERR_OK ) +#endif + { + return error; + } + } + + /* Render subframe */ +#ifdef FIX_356_ISM_METADATA_SYNC + if ( ( error = TDREND_GetMix( hBinRendererTd, output, subframe_length, subframe_idx, ism_md_subframe_update ) ) != IVAS_ERR_OK ) +#else + if ( ( error = TDREND_GetMix( hBinRendererTd, output, subframe_length, subframe_idx ) ) != IVAS_ERR_OK ) +#endif + { + return error; + } + } + + + if ( hReverb != NULL && hReverb->pConfig.roomAcoustics.late_reverb_on ) + { + /* add reverb to rendered signals */ + v_add( reverb_signal[0], output[0], output[0], output_frame ); + v_add( reverb_signal[1], output[1], output[1], output_frame ); + } + + return IVAS_ERR_OK; +} + + +/*---------------------------------------------------------------------* + * TDREND_GetMix() + * + * Render one 5 ms subframe from the mixer + *---------------------------------------------------------------------*/ + +ivas_error TDREND_GetMix( + BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD renderer handle */ +#ifdef JBM_TSM_ON_TCS + float *output[], /* i/o: ISM object synth / rendered output in 0,1 */ +#else + float output[][L_FRAME48k], /* i/o: ISM object synth / rendered output in 0,1 */ +#endif + const int16_t subframe_length, /* i/o: subframe length */ + const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ +#ifdef FIX_356_ISM_METADATA_SYNC + , + const int16_t ism_md_subframe_update /* Number of subframes to delay ism metadata to sync with audio */ +#endif +) +{ + int16_t i; + TDREND_SRC_t *Src_p; + TDREND_SRC_SPATIAL_t *SrcSpatial_p; + TDREND_SRC_REND_t *SrcRend_p; + ivas_error error; + float output_buf[2][L_SPATIAL_SUBFR_48k]; /* Temp buffer for left/right rendered signal */ + float hrf_left_delta[SFX_SPAT_BIN_MAX_FILTER_LENGTH]; + float hrf_right_delta[SFX_SPAT_BIN_MAX_FILTER_LENGTH]; + int16_t intp_count; +#ifdef ISM_NON_DIEGETIC_PAN + float pan_left, pan_right; +#endif +#ifdef FIX_356_ISM_METADATA_SYNC + int16_t subframe_update_flag; + subframe_update_flag = subframe_idx == ism_md_subframe_update; +#endif + error = IVAS_ERR_OK; + + /* Clear the output buffer to accumulate rendered sources */ + set_f( output_buf[0], 0.0f, subframe_length ); + set_f( output_buf[1], 0.0f, subframe_length ); + + /* Clear interpolation buffers and counter */ + set_f( hrf_left_delta, 0.0f, SFX_SPAT_BIN_MAX_FILTER_LENGTH ); + set_f( hrf_right_delta, 0.0f, SFX_SPAT_BIN_MAX_FILTER_LENGTH ); + intp_count = 0; + + /* Create the mix */ + /* Loop through the source list and render each source */ + for ( i = 0; i < hBinRendererTd->NumOfSrcs; i++ ) + { + Src_p = hBinRendererTd->Sources[i]; + SrcSpatial_p = Src_p->SrcSpatial_p; + SrcRend_p = Src_p->SrcRend_p; + + /* Update rendering params if needed */ +#ifdef ISM_NON_DIEGETIC_PAN + if ( ( SrcRend_p->PlayStatus == TDREND_PLAYSTATUS_PLAYING ) && ( hBinRendererTd->Listener_p->PoseUpdated || SrcSpatial_p->Updated ) ) +#else + if ( hBinRendererTd->Listener_p->PoseUpdated || SrcSpatial_p->Updated ) +#endif + { + TDREND_SRC_REND_UpdateFiltersFromSpatialParams( hBinRendererTd, SrcRend_p, SrcSpatial_p, Src_p->hrf_left_prev, + Src_p->hrf_right_prev, hrf_left_delta, hrf_right_delta, &intp_count, &Src_p->filterlength, &Src_p->itd, &Src_p->Gain, Src_p, +#ifdef FIX_356_ISM_METADATA_SYNC + subframe_update_flag +#else + subframe_idx +#endif + ); + } + + /* Render source if needed */ + if ( ( SrcRend_p->InputAvailable == TRUE ) && ( SrcRend_p->PlayStatus == TDREND_PLAYSTATUS_PLAYING ) ) + { + error = TDREND_REND_RenderSourceHRFilt( Src_p, hrf_left_delta, hrf_right_delta, intp_count, output_buf, subframe_length ); + } + +#ifdef ISM_NON_DIEGETIC_PAN + if ( ( SrcRend_p->InputAvailable == TRUE ) && ( SrcRend_p->PlayStatus == TDREND_PLAYSTATUS_PLAYING_NON_DIEGETIC ) ) + { + pan_left = ( SrcSpatial_p->Pos_p[1] + 1.f ) * 0.5f; + pan_right = 1.f - pan_left; + v_multc_acc( &Src_p->InputFrame_p[subframe_idx * subframe_length], pan_left, output_buf[0], subframe_length ); + v_multc_acc( &Src_p->InputFrame_p[subframe_idx * subframe_length], pan_right, output_buf[1], subframe_length ); + } +#endif + } + + /* Populate output variable */ + mvr2r( output_buf[0], output[0] + subframe_idx * subframe_length, subframe_length ); /* Left */ + mvr2r( output_buf[1], output[1] + subframe_idx * subframe_length, subframe_length ); /* Right */ + + /* Clear the PoseUpdated and Source position update flags */ + TDREND_Clear_Update_flags( hBinRendererTd ); + + return error; +} + + +/*---------------------------------------------------------------------* + * TDREND_Clear_Update_flags() + * + * Initializes the audio mixer module + *---------------------------------------------------------------------*/ + +static void TDREND_Clear_Update_flags( + BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd /* i/o: TD renderer handle */ +) +{ + int16_t i; + + hBinRendererTd->Listener_p->PoseUpdated = FALSE; + + for ( i = 0; i < hBinRendererTd->NumOfSrcs; i++ ) + { + hBinRendererTd->Sources[i]->SrcSpatial_p->Updated = FALSE; + } + + return; +} + + +/*---------------------------------------------------------------------* + * TDREND_Update_object_positions() + * + * Update object position(s) + *---------------------------------------------------------------------*/ + +void TDREND_Update_object_positions( + BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o : TD Renderer handle */ + const int16_t num_src, /* i : number of sources to render */ +#ifndef FIX_356_ISM_METADATA_SYNC + const int16_t lfe_idx, /* i : Input LFE index */ +#endif + const IVAS_FORMAT in_format, /* i : Format of input sources */ + const ISM_METADATA_HANDLE *hIsmMetaData /* i : Input metadata for ISM objects */ +#ifndef FIX_356_ISM_METADATA_SYNC + , +#ifdef JBM_TSM_ON_TCS + float *output[] +#else + float output[][L_FRAME48k] /* i/o: SCE/MC channels */ +#endif +#endif +) +{ + TDREND_DirAtten_t *DirAtten_p; + int16_t nS; + float Pos[3]; + float Dir[3]; +#ifndef FIX_356_ISM_METADATA_SYNC + int16_t c_indx; +#endif + + DirAtten_p = hBinRendererTd->DirAtten_p; + + /* For each source, write the frame data to the source object*/ +#ifndef FIX_356_ISM_METADATA_SYNC + c_indx = 0; +#endif + for ( nS = 0; nS < num_src; nS++ ) + { +#ifndef FIX_356_ISM_METADATA_SYNC + if ( !( in_format == MC_FORMAT && nS == lfe_idx ) ) /* Skip LFE for MC */ + { + hBinRendererTd->Sources[c_indx]->InputFrame_p = output[nS]; + hBinRendererTd->Sources[c_indx]->SrcRend_p->InputAvailable = TRUE; + c_indx++; + } +#endif + if ( in_format == ISM_FORMAT ) + { + /* Update the source positions */ + /* Source position and direction */ + angles_to_vec( hIsmMetaData[nS]->radius, hIsmMetaData[nS]->azimuth, hIsmMetaData[nS]->elevation, Pos ); + angles_to_vec( 1.0f, hIsmMetaData[nS]->yaw, hIsmMetaData[nS]->pitch, Dir ); + +#ifndef FIX_463_TD_RENDERER_DIRECTIVITY_RESET + + /* Source directivity info */ + DirAtten_p->ConeInnerAngle = 360.0f; + DirAtten_p->ConeOuterAngle = 360.0f; + DirAtten_p->ConeOuterGain = 1.0f; +#endif + + TDREND_MIX_SRC_SetPos( hBinRendererTd, nS, Pos ); + TDREND_MIX_SRC_SetDirAtten( hBinRendererTd, nS, DirAtten_p ); + +#ifdef ISM_NON_DIEGETIC_PAN + if ( hIsmMetaData[nS]->non_diegetic_flag ) + { + Pos[0] = 0; + Pos[1] = hIsmMetaData[nS]->azimuth / 90.f; + Pos[2] = 0; + TDREND_MIX_SRC_SetPos( hBinRendererTd, nS, Pos ); + TDREND_MIX_SRC_SetPlayState( hBinRendererTd, nS, TDREND_PLAYSTATUS_PLAYING_NON_DIEGETIC ); + } + else + { + TDREND_MIX_SRC_SetPlayState( hBinRendererTd, nS, TDREND_PLAYSTATUS_PLAYING ); + } +#else + TDREND_MIX_SRC_SetPlayState( hBinRendererTd, nS, TDREND_PLAYSTATUS_PLAYING ); +#endif + TDREND_MIX_SRC_SetDir( hBinRendererTd, nS, Dir ); + } + } + + return; +} + + +/*---------------------------------------------------------------------* + * TDREND_Update_listener_orientation() + * + * Update listener orientation (s) + *---------------------------------------------------------------------*/ + +void TDREND_Update_listener_orientation( + BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD Renderer handle */ + const int16_t headRotEnabled, /* i : Headrotation flag */ + const IVAS_QUATERNION *headPosition, /* i : Listener orientation */ + const IVAS_VECTOR3 *Pos /* i : Listener Position */ +) +{ + float FrontVec[3]; + float UpVec[3]; + float Rmat[3][3]; + float Pos_p[3]; + + if ( headRotEnabled ) + { + /* Obtain head rotation matrix */ + QuatToRotMat( *headPosition, Rmat ); + /* Apply rotation matrix to looking vector [1;0;0] */ + FrontVec[0] = Rmat[0][0]; + FrontVec[1] = Rmat[0][1]; + FrontVec[2] = Rmat[0][2]; + /* Apply rotation matrix to up vector [0;0;1] */ + UpVec[0] = Rmat[2][0]; + UpVec[1] = Rmat[2][1]; + UpVec[2] = Rmat[2][2]; + /* Input position */ + Pos_p[0] = ( *Pos ).x; + Pos_p[1] = ( *Pos ).y; + Pos_p[2] = ( *Pos ).z; + } + else + { + /* Oriented with looking vector along the x axis */ + FrontVec[0] = 1.0f; + FrontVec[1] = 0.0f; + FrontVec[2] = 0.0f; + /* Oriented with up vector along the z axis */ + UpVec[0] = 0.0f; + UpVec[1] = 0.0f; + UpVec[2] = 1.0f; + /* Listener at the origin */ + Pos_p[0] = 0.0f; + Pos_p[1] = 0.0f; + Pos_p[2] = 0.0f; + } + + /* Set the listener position and orientation:*/ + TDREND_MIX_LIST_SetPos( hBinRendererTd, Pos_p ); + TDREND_MIX_LIST_SetOrient( hBinRendererTd, FrontVec, UpVec ); + + return; +} + + +/*---------------------------------------------------------------------* + * ivas_td_binaural_open_ext() + * + * + *---------------------------------------------------------------------*/ + +ivas_error ivas_td_binaural_open_ext( + TDREND_WRAPPER *pTDRend, + IVAS_REND_AudioConfig inConfig, + RENDER_CONFIG_DATA *hRendCfg, /* i : Renderer configuration */ + LSSETUP_CUSTOM_STRUCT *customLsInput, + const int32_t outFs ) +{ + int16_t nchan_transport; + AUDIO_CONFIG transport_config; + IVAS_FORMAT ivas_format; + IVAS_OUTPUT_SETUP hTransSetup; + ivas_error error; + float *directivity = NULL; + + if ( inConfig != IVAS_REND_AUDIO_CONFIG_LS_CUSTOM ) + { + if ( ( error = getAudioConfigNumChannels( inConfig, &nchan_transport ) ) != IVAS_ERR_OK ) + { + return error; + } + } + else + { + nchan_transport = customLsInput->num_spk; + } + + transport_config = getIvasAudioConfigFromRendAudioConfig( inConfig ); + ivas_format = ( getAudioConfigType( inConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) ? MC_FORMAT : ISM_FORMAT; + hTransSetup.ls_azimuth = customLsInput->ls_azimuth; + hTransSetup.ls_elevation = customLsInput->ls_elevation; + + if ( NULL != hRendCfg ) + { + directivity = hRendCfg->directivity; + } + + return ivas_td_binaural_open_unwrap( &pTDRend->hHrtfTD, outFs, nchan_transport, ivas_format, transport_config, directivity, hTransSetup, &pTDRend->hBinRendererTd, &pTDRend->binaural_latency_ns ); +} + + +/*---------------------------------------------------------------------* + * ivas_td_binaural_renderer_ext() + * + * Receives the current frames for the object streams, updates metadata + * and renders the current frame. + *---------------------------------------------------------------------*/ + +ivas_error ivas_td_binaural_renderer_ext( + const TDREND_WRAPPER *pTDRend, /* i : TD Renderer wrapper structure */ + const IVAS_REND_AudioConfig inConfig, /* i : Input audio configuration */ + const LSSETUP_CUSTOM_STRUCT *customLsInput, /* i : Input custom loudspeaker layout */ + const IVAS_REND_HeadRotData *headRotData, /* i : Input head positions */ + const IVAS_REND_AudioObjectPosition *currentPos, /* i : Object position */ + const REVERB_HANDLE hReverb, /* i : Reverberator handle */ + const int16_t output_frame, /* i : output frame length */ + float output[][L_FRAME48k] /* i/o: SCE channels / Binaural synthesis */ +) +{ + ISM_METADATA_FRAME hIsmMetaDataFrame; + ISM_METADATA_HANDLE hIsmMetaData[1]; + int16_t lfe_idx; + int16_t num_src; + IVAS_FORMAT ivas_format; + IVAS_REND_AudioConfigType inConfigType; + AUDIO_CONFIG transport_config; + ivas_error error; +#ifdef FIX_356_ISM_METADATA_SYNC + int16_t ism_md_subframe_update = 0; +#endif +#ifdef JBM_TSM_ON_TCS + float *p_output[MAX_OUTPUT_CHANNELS]; + int16_t ch; + + for ( ch = 0; ch < MAX_OUTPUT_CHANNELS; ch++ ) + { + p_output[ch] = output[ch]; + } +#endif + + push_wmops( "ivas_td_binaural_renderer_ext" ); + + inConfigType = getAudioConfigType( inConfig ); + lfe_idx = LFE_CHANNEL; + hIsmMetaData[0] = NULL; + + if ( inConfigType == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) + { + ivas_format = MC_FORMAT; + transport_config = getIvasAudioConfigFromRendAudioConfig( inConfig ); + if ( inConfig != IVAS_REND_AUDIO_CONFIG_LS_CUSTOM ) + { + if ( ( error = getAudioConfigNumChannels( inConfig, &num_src ) ) != IVAS_ERR_OK ) + { + return error; + } + } + else + { + lfe_idx = ( customLsInput->num_lfe > 0 ) ? customLsInput->lfe_idx[0] : -1; + num_src = customLsInput->num_spk + customLsInput->num_lfe; + } + } + else + { + ivas_format = ISM_FORMAT; + num_src = 1; + transport_config = AUDIO_CONFIG_ISM1; + hIsmMetaData[0] = &hIsmMetaDataFrame; + hIsmMetaData[0]->azimuth = currentPos->azimuth; + hIsmMetaData[0]->elevation = currentPos->elevation; + hIsmMetaData[0]->yaw = currentPos->yaw; + hIsmMetaData[0]->pitch = currentPos->pitch; + hIsmMetaData[0]->radius = currentPos->radius; +#ifdef ISM_NON_DIEGETIC_PAN + hIsmMetaData[0]->non_diegetic_flag = currentPos->non_diegetic_flag; +#endif + } + +#ifdef JBM_TSM_ON_TCS +#ifdef FIX_356_ISM_METADATA_SYNC + if( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, + ( headRotData != NULL ) ? headRotData->headPositions : NULL, + ( headRotData != NULL ) ? headRotData->Pos : NULL, ism_md_subframe_update, p_output, output_frame ) ) != IVAS_ERR_OK ) +#else + if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, + ( headRotData != NULL ) ? headRotData->headPositions : NULL, + ( headRotData != NULL ) ? headRotData->Pos : NULL, p_output, output_frame ) ) != IVAS_ERR_OK ) +#endif +#else +#ifdef FIX_356_ISM_METADATA_SYNC + if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, + ( headRotData != NULL ) ? headRotData->headPositions : NULL, + ( headRotData != NULL ) ? headRotData->Pos : NULL, ism_md_subframe_update, output, output_frame ) ) != IVAS_ERR_OK ) +#else + if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, + ( headRotData != NULL ) ? headRotData->headPositions : NULL, + ( headRotData != NULL ) ? headRotData->Pos : NULL, output, output_frame ) ) != IVAS_ERR_OK ) +#endif +#endif + { + return error; + } + + pop_wmops(); + + return IVAS_ERR_OK; +} + +/*---------------------------------------------------------------------* + * angles_to_vec() + * + * Convert azimuth and elevation angles to position/orientation vector + *---------------------------------------------------------------------*/ + +static void angles_to_vec( + const float radius, /* i : radius */ + const float azimuth, /* i : Azimuth angle */ + const float elevation, /* i : Elevation angle */ + float *vec /* o : Pos/Dir vector */ +) +{ + vec[0] = radius * cosf( elevation * PI_OVER_180 ) * cosf( azimuth * PI_OVER_180 ); + vec[1] = radius * cosf( elevation * PI_OVER_180 ) * sinf( azimuth * PI_OVER_180 ); + vec[2] = radius * sinf( elevation * PI_OVER_180 ); + + return; +} -- GitLab From 29758324c77b07bda930d4a764bbad566fd250da Mon Sep 17 00:00:00 2001 From: Sumeyra Kanik Date: Wed, 17 May 2023 17:45:54 +0200 Subject: [PATCH 14/18] Fix crash Formatting issue --- lib_rend/ivas_objectRenderer.c | 805 --------------------------------- 1 file changed, 805 deletions(-) diff --git a/lib_rend/ivas_objectRenderer.c b/lib_rend/ivas_objectRenderer.c index 0eec81b692..3db591d094 100644 --- a/lib_rend/ivas_objectRenderer.c +++ b/lib_rend/ivas_objectRenderer.c @@ -803,808 +803,3 @@ static void angles_to_vec( return; } -/****************************************************************************************************** - - (C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. - -*******************************************************************************************************/ - -#include "ivas_stat_rend.h" -#include -#include "options.h" -#include "prot.h" -#ifdef ISM_NON_DIEGETIC_PAN -#include "ivas_prot.h" -#endif -#include "ivas_prot_rend.h" -#include -#include "ivas_rom_com.h" -#ifdef DEBUGGING -#include "debug.h" -#endif -#include "wmc_auto.h" - - -/*---------------------------------------------------------------------* - * Local function prototypes - *---------------------------------------------------------------------*/ - - -static void TDREND_Clear_Update_flags( BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd ); - -static void angles_to_vec( const float radius, const float azimuth, const float elevation, float *vec ); - - -/*---------------------------------------------------------------------* - * ivas_td_binaural_open_unwrap() - * - * Call TD open/init function without st_ivas - *---------------------------------------------------------------------*/ - -ivas_error ivas_td_binaural_open_unwrap( - TDREND_HRFILT_FiltSet_t **hHrtfTD, /* i/o: HR filter model (from file or NULL) */ - const int32_t output_Fs, /* i : Output sampling rate */ - const int16_t nchan_transport, /* i : Number of channels */ - const IVAS_FORMAT ivas_format, /* i : IVAS format (ISM/MC) */ - const AUDIO_CONFIG transport_config, /* i : Transport configuration */ - const float *directivity, /* i : Directivity pattern (used for ISM) */ - const IVAS_OUTPUT_SETUP hTransSetup, /* i : Loudspeaker layout */ - BINAURAL_TD_OBJECT_RENDERER_HANDLE *hBinRendererTd, /* o : TD renderer handle */ - int32_t *binaural_latency_ns /* i : Binauralization delay */ -) -{ - BINAURAL_TD_OBJECT_RENDERER_HANDLE pBinRendTd; - TDREND_PosType_t PosType; - int16_t nS; - int16_t SrcInd[MAX_NUM_TDREND_CHANNELS]; - const float *ls_azimuth, *ls_elevation; - float Pos[3]; - float Dir[3]; - TDREND_DirAtten_t *DirAtten_p; - int16_t nchan_rend; - ivas_error error; - - error = IVAS_ERR_OK; - - if ( ( pBinRendTd = malloc( sizeof( BINAURAL_TD_OBJECT_RENDERER ) ) ) == NULL ) - { - return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for TD renderer\n" ) ); - } - if ( ( pBinRendTd->TdRend_MixSpatSpec_p = malloc( sizeof( TDREND_MixSpatSpec_t ) ) ) == NULL ) - { - return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for TD renderer\n" ) ); - } - if ( ( pBinRendTd->DirAtten_p = malloc( sizeof( TDREND_DirAtten_t ) ) ) == NULL ) - { - return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for TD renderer\n" ) ); - } - if ( ( pBinRendTd->Listener_p = malloc( sizeof( TDREND_MIX_Listener_t ) ) ) == NULL ) - { - return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for TD renderer\n" ) ); - } - - pBinRendTd->NumOfSrcs = 0; - pBinRendTd->MaxSrcInd = -1; - - /* Mixer spatial setup */ - pBinRendTd->TdRend_MixSpatSpec_p->UseCommonDistAttenModel = TRUE; - pBinRendTd->TdRend_MixSpatSpec_p->DistAttenModel = 0; /* 0=Turned off, else use TDREND_DIST_ATTEN_MODEL_INV_DIST_CLAMPED */ - - if ( ( error = TDREND_MIX_Init( pBinRendTd, hHrtfTD, pBinRendTd->TdRend_MixSpatSpec_p, output_Fs ) ) != IVAS_ERR_OK ) - { - return error; - } - - /* Set the attenuation (or can set MixSpatSpec.DistAttenModel above) */ - if ( ( error = TDREND_MIX_SetDistAttenModel( pBinRendTd, TDREND_DIST_ATTEN_MODEL_INV_DIST_CLAMPED ) ) != IVAS_ERR_OK ) - { - return error; - } - - /* Add sources to module and mixer, headphones */ - PosType = TDREND_POSTYPE_ABSOLUTE; /* or TDREND_POSTYPE_RELATIVE_TO_LISTENER */ - - nchan_rend = nchan_transport; - if ( ( ivas_format == MC_FORMAT ) && ( transport_config != AUDIO_CONFIG_LS_CUSTOM ) ) - { - nchan_rend--; /* Skip LFE channel -- added to the others */ - } - - for ( nS = 0; nS < nchan_rend; nS++ ) - { - if ( ( error = TDREND_MIX_AddSrc( pBinRendTd, &SrcInd[nS], PosType ) ) != IVAS_ERR_OK ) - { - return error; - } - } - - if ( ivas_format == MC_FORMAT ) - { - switch ( transport_config ) - { - case AUDIO_CONFIG_5_1: - ls_azimuth = ls_azimuth_CICP6; - ls_elevation = ls_elevation_CICP6; - break; - case AUDIO_CONFIG_7_1: - ls_azimuth = ls_azimuth_CICP12; - ls_elevation = ls_elevation_CICP12; - break; - case AUDIO_CONFIG_5_1_2: - ls_azimuth = ls_azimuth_CICP14; - ls_elevation = ls_elevation_CICP14; - break; - case AUDIO_CONFIG_5_1_4: - ls_azimuth = ls_azimuth_CICP16; - ls_elevation = ls_elevation_CICP16; - break; - case AUDIO_CONFIG_7_1_4: - ls_azimuth = ls_azimuth_CICP19; - ls_elevation = ls_elevation_CICP19; - break; - case AUDIO_CONFIG_LS_CUSTOM: - ls_azimuth = hTransSetup.ls_azimuth; - ls_elevation = hTransSetup.ls_elevation; - break; - default: - ls_azimuth = NULL; - ls_elevation = NULL; - } - - DirAtten_p = pBinRendTd->DirAtten_p; - - for ( nS = 0; nS < nchan_rend; nS++ ) - { - /* Set source positions according to loudspeaker layout */ - angles_to_vec( 1.0f, ls_azimuth[nS], ls_elevation[nS], Pos ); - Dir[0] = 1.0f; - Dir[1] = 0.0f; - Dir[2] = 0.0f; - - /* Source directivity info */ - DirAtten_p->ConeInnerAngle = 360.0f; - DirAtten_p->ConeOuterAngle = 360.0f; - DirAtten_p->ConeOuterGain = 1.0f; - - TDREND_MIX_SRC_SetPos( pBinRendTd, nS, Pos ); - TDREND_MIX_SRC_SetDir( pBinRendTd, nS, Dir ); - TDREND_MIX_SRC_SetPlayState( pBinRendTd, nS, TDREND_PLAYSTATUS_PLAYING ); - TDREND_MIX_SRC_SetDirAtten( pBinRendTd, nS, DirAtten_p ); - } - } - if ( ivas_format == ISM_FORMAT ) - { - DirAtten_p = pBinRendTd->DirAtten_p; - if ( NULL == directivity ) - { - DirAtten_p->ConeInnerAngle = 360.0f; /* Front cone */ - DirAtten_p->ConeOuterAngle = 360.0f; /* Back cone */ - DirAtten_p->ConeOuterGain = 1.0f; /* Back attenuation */ - } - else - { - DirAtten_p->ConeInnerAngle = directivity[0]; - DirAtten_p->ConeOuterAngle = directivity[1]; - DirAtten_p->ConeOuterGain = directivity[2]; - } - - for ( nS = 0; nS < nchan_rend; nS++ ) - { - TDREND_MIX_SRC_SetDirAtten( pBinRendTd, nS, DirAtten_p ); - } - } - - *hBinRendererTd = pBinRendTd; - *binaural_latency_ns = (int32_t) ( ( *hBinRendererTd )->HrFiltSet_p->latency_s * 1000000000.f ); - - return error; -} - - -/*---------------------------------------------------------------------* - * ivas_td_binaural_close() - * - * Close TD Object binaural renderer - *---------------------------------------------------------------------*/ - -void ivas_td_binaural_close( - BINAURAL_TD_OBJECT_RENDERER_HANDLE *hBinRendererTd /* i/o: TD binaural object renderer handle */ -) -{ - if ( hBinRendererTd == NULL || *hBinRendererTd == NULL ) - { - return; - } - - free( ( *hBinRendererTd )->TdRend_MixSpatSpec_p ); - free( ( *hBinRendererTd )->DirAtten_p ); - - TDREND_MIX_Dealloc( *hBinRendererTd ); - - free( *hBinRendererTd ); - *hBinRendererTd = NULL; - - return; -} - -/*---------------------------------------------------------------------* - * ivas_td_binaural_renderer_unwrap() - * - * Call ivas_td_binaural_renderer() without st_ivas. - *---------------------------------------------------------------------*/ - -ivas_error ivas_td_binaural_renderer_unwrap( - const REVERB_HANDLE hReverb, /* i : Reverberator handle */ - const AUDIO_CONFIG transport_config, /* i : Transport configuration */ - BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD binaural object renderer handle */ - const int16_t num_src, /* i : number of sources to render */ - const int16_t lfe_idx, /* i : LFE channel index */ - const IVAS_FORMAT ivas_format, /* i : IVAS format */ - ISM_METADATA_HANDLE *hIsmMetaData, /* i : ISM metadata handle */ - const int16_t Opt_Headrotation, /* i : Head rotation flag */ - const IVAS_QUATERNION *Quaternions, /* i : Head tracking data per subframe */ - const IVAS_VECTOR3 *Pos, /* i : Listener position data per subframe */ -#ifdef FIX_356_ISM_METADATA_SYNC - const int16_t ism_md_subframe_update, /* i: Number of subframes to delay ism metadata to sync with audio */ -#endif -#ifdef JBM_TSM_ON_TCS - float *output[], /* i/o: SCE channels / Binaural synthesis */ -#else - float output[][L_FRAME48k], /* i/o: SCE channels / Binaural synthesis */ -#endif - const int16_t output_frame /* i : output frame length */ -) -{ - int16_t subframe_length; - int16_t subframe_idx; - float reverb_signal[BINAURAL_CHANNELS][L_FRAME48k]; - ivas_error error; -#ifdef JBM_TSM_ON_TCS - float *p_reverb_signal[BINAURAL_CHANNELS]; - int16_t ch; - - for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) - { - p_reverb_signal[ch] = reverb_signal[ch]; - } -#endif - - subframe_length = output_frame / MAX_PARAM_SPATIAL_SUBFRAMES; - -#ifdef FIX_356_ISM_METADATA_SYNC - int16_t c_indx, nS; - c_indx = 0; - - for ( nS = 0; nS < num_src; nS++ ) - { - if ( !( ivas_format == MC_FORMAT && nS == lfe_idx ) ) /* Skip LFE for MC */ - { - hBinRendererTd->Sources[c_indx]->InputFrame_p = output[nS]; - hBinRendererTd->Sources[c_indx]->SrcRend_p->InputAvailable = TRUE; - c_indx++; - } - } -#else - /* Update object position(s) */ - TDREND_Update_object_positions( hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, output ); -#endif - - for ( subframe_idx = 0; subframe_idx < MAX_PARAM_SPATIAL_SUBFRAMES; subframe_idx++ ) - { -#ifdef FIX_356_ISM_METADATA_SYNC - if ( subframe_idx == ism_md_subframe_update ) - { - /* Update object position(s) */ - TDREND_Update_object_positions( hBinRendererTd, num_src, ivas_format, hIsmMetaData ); - } -#endif - /* Update the listener's location/orientation */ - TDREND_Update_listener_orientation( hBinRendererTd, Opt_Headrotation, ( Quaternions != NULL ) ? &Quaternions[subframe_idx] : NULL, ( Pos != NULL ) ? &Pos[subframe_idx] : NULL ); - - if ( hReverb != NULL && hReverb->pConfig.roomAcoustics.late_reverb_on ) - { -#ifdef JBM_TSM_ON_TCS - if ( ( error = ivas_reverb_process( hReverb, transport_config, 0, output, p_reverb_signal, subframe_idx ) ) != IVAS_ERR_OK ) -#else - if ( ( error = ivas_reverb_process( hReverb, transport_config, 0, output, reverb_signal, subframe_idx ) ) != IVAS_ERR_OK ) -#endif - { - return error; - } - } - - /* Render subframe */ -#ifdef FIX_356_ISM_METADATA_SYNC - if ( ( error = TDREND_GetMix( hBinRendererTd, output, subframe_length, subframe_idx, ism_md_subframe_update ) ) != IVAS_ERR_OK ) -#else - if ( ( error = TDREND_GetMix( hBinRendererTd, output, subframe_length, subframe_idx ) ) != IVAS_ERR_OK ) -#endif - { - return error; - } - } - - - if ( hReverb != NULL && hReverb->pConfig.roomAcoustics.late_reverb_on ) - { - /* add reverb to rendered signals */ - v_add( reverb_signal[0], output[0], output[0], output_frame ); - v_add( reverb_signal[1], output[1], output[1], output_frame ); - } - - return IVAS_ERR_OK; -} - - -/*---------------------------------------------------------------------* - * TDREND_GetMix() - * - * Render one 5 ms subframe from the mixer - *---------------------------------------------------------------------*/ - -ivas_error TDREND_GetMix( - BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD renderer handle */ -#ifdef JBM_TSM_ON_TCS - float *output[], /* i/o: ISM object synth / rendered output in 0,1 */ -#else - float output[][L_FRAME48k], /* i/o: ISM object synth / rendered output in 0,1 */ -#endif - const int16_t subframe_length, /* i/o: subframe length */ - const int16_t subframe_idx /* i : Subframe index to 5 ms subframe */ -#ifdef FIX_356_ISM_METADATA_SYNC - , - const int16_t ism_md_subframe_update /* Number of subframes to delay ism metadata to sync with audio */ -#endif -) -{ - int16_t i; - TDREND_SRC_t *Src_p; - TDREND_SRC_SPATIAL_t *SrcSpatial_p; - TDREND_SRC_REND_t *SrcRend_p; - ivas_error error; - float output_buf[2][L_SPATIAL_SUBFR_48k]; /* Temp buffer for left/right rendered signal */ - float hrf_left_delta[SFX_SPAT_BIN_MAX_FILTER_LENGTH]; - float hrf_right_delta[SFX_SPAT_BIN_MAX_FILTER_LENGTH]; - int16_t intp_count; -#ifdef ISM_NON_DIEGETIC_PAN - float pan_left, pan_right; -#endif -#ifdef FIX_356_ISM_METADATA_SYNC - int16_t subframe_update_flag; - subframe_update_flag = subframe_idx == ism_md_subframe_update; -#endif - error = IVAS_ERR_OK; - - /* Clear the output buffer to accumulate rendered sources */ - set_f( output_buf[0], 0.0f, subframe_length ); - set_f( output_buf[1], 0.0f, subframe_length ); - - /* Clear interpolation buffers and counter */ - set_f( hrf_left_delta, 0.0f, SFX_SPAT_BIN_MAX_FILTER_LENGTH ); - set_f( hrf_right_delta, 0.0f, SFX_SPAT_BIN_MAX_FILTER_LENGTH ); - intp_count = 0; - - /* Create the mix */ - /* Loop through the source list and render each source */ - for ( i = 0; i < hBinRendererTd->NumOfSrcs; i++ ) - { - Src_p = hBinRendererTd->Sources[i]; - SrcSpatial_p = Src_p->SrcSpatial_p; - SrcRend_p = Src_p->SrcRend_p; - - /* Update rendering params if needed */ -#ifdef ISM_NON_DIEGETIC_PAN - if ( ( SrcRend_p->PlayStatus == TDREND_PLAYSTATUS_PLAYING ) && ( hBinRendererTd->Listener_p->PoseUpdated || SrcSpatial_p->Updated ) ) -#else - if ( hBinRendererTd->Listener_p->PoseUpdated || SrcSpatial_p->Updated ) -#endif - { - TDREND_SRC_REND_UpdateFiltersFromSpatialParams( hBinRendererTd, SrcRend_p, SrcSpatial_p, Src_p->hrf_left_prev, - Src_p->hrf_right_prev, hrf_left_delta, hrf_right_delta, &intp_count, &Src_p->filterlength, &Src_p->itd, &Src_p->Gain, Src_p, -#ifdef FIX_356_ISM_METADATA_SYNC - subframe_update_flag -#else - subframe_idx -#endif - ); - } - - /* Render source if needed */ - if ( ( SrcRend_p->InputAvailable == TRUE ) && ( SrcRend_p->PlayStatus == TDREND_PLAYSTATUS_PLAYING ) ) - { - error = TDREND_REND_RenderSourceHRFilt( Src_p, hrf_left_delta, hrf_right_delta, intp_count, output_buf, subframe_length ); - } - -#ifdef ISM_NON_DIEGETIC_PAN - if ( ( SrcRend_p->InputAvailable == TRUE ) && ( SrcRend_p->PlayStatus == TDREND_PLAYSTATUS_PLAYING_NON_DIEGETIC ) ) - { - pan_left = ( SrcSpatial_p->Pos_p[1] + 1.f ) * 0.5f; - pan_right = 1.f - pan_left; - v_multc_acc( &Src_p->InputFrame_p[subframe_idx * subframe_length], pan_left, output_buf[0], subframe_length ); - v_multc_acc( &Src_p->InputFrame_p[subframe_idx * subframe_length], pan_right, output_buf[1], subframe_length ); - } -#endif - } - - /* Populate output variable */ - mvr2r( output_buf[0], output[0] + subframe_idx * subframe_length, subframe_length ); /* Left */ - mvr2r( output_buf[1], output[1] + subframe_idx * subframe_length, subframe_length ); /* Right */ - - /* Clear the PoseUpdated and Source position update flags */ - TDREND_Clear_Update_flags( hBinRendererTd ); - - return error; -} - - -/*---------------------------------------------------------------------* - * TDREND_Clear_Update_flags() - * - * Initializes the audio mixer module - *---------------------------------------------------------------------*/ - -static void TDREND_Clear_Update_flags( - BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd /* i/o: TD renderer handle */ -) -{ - int16_t i; - - hBinRendererTd->Listener_p->PoseUpdated = FALSE; - - for ( i = 0; i < hBinRendererTd->NumOfSrcs; i++ ) - { - hBinRendererTd->Sources[i]->SrcSpatial_p->Updated = FALSE; - } - - return; -} - - -/*---------------------------------------------------------------------* - * TDREND_Update_object_positions() - * - * Update object position(s) - *---------------------------------------------------------------------*/ - -void TDREND_Update_object_positions( - BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o : TD Renderer handle */ - const int16_t num_src, /* i : number of sources to render */ -#ifndef FIX_356_ISM_METADATA_SYNC - const int16_t lfe_idx, /* i : Input LFE index */ -#endif - const IVAS_FORMAT in_format, /* i : Format of input sources */ - const ISM_METADATA_HANDLE *hIsmMetaData /* i : Input metadata for ISM objects */ -#ifndef FIX_356_ISM_METADATA_SYNC - , -#ifdef JBM_TSM_ON_TCS - float *output[] -#else - float output[][L_FRAME48k] /* i/o: SCE/MC channels */ -#endif -#endif -) -{ - TDREND_DirAtten_t *DirAtten_p; - int16_t nS; - float Pos[3]; - float Dir[3]; -#ifndef FIX_356_ISM_METADATA_SYNC - int16_t c_indx; -#endif - - DirAtten_p = hBinRendererTd->DirAtten_p; - - /* For each source, write the frame data to the source object*/ -#ifndef FIX_356_ISM_METADATA_SYNC - c_indx = 0; -#endif - for ( nS = 0; nS < num_src; nS++ ) - { -#ifndef FIX_356_ISM_METADATA_SYNC - if ( !( in_format == MC_FORMAT && nS == lfe_idx ) ) /* Skip LFE for MC */ - { - hBinRendererTd->Sources[c_indx]->InputFrame_p = output[nS]; - hBinRendererTd->Sources[c_indx]->SrcRend_p->InputAvailable = TRUE; - c_indx++; - } -#endif - if ( in_format == ISM_FORMAT ) - { - /* Update the source positions */ - /* Source position and direction */ - angles_to_vec( hIsmMetaData[nS]->radius, hIsmMetaData[nS]->azimuth, hIsmMetaData[nS]->elevation, Pos ); - angles_to_vec( 1.0f, hIsmMetaData[nS]->yaw, hIsmMetaData[nS]->pitch, Dir ); - -#ifndef FIX_463_TD_RENDERER_DIRECTIVITY_RESET - - /* Source directivity info */ - DirAtten_p->ConeInnerAngle = 360.0f; - DirAtten_p->ConeOuterAngle = 360.0f; - DirAtten_p->ConeOuterGain = 1.0f; -#endif - - TDREND_MIX_SRC_SetPos( hBinRendererTd, nS, Pos ); - TDREND_MIX_SRC_SetDirAtten( hBinRendererTd, nS, DirAtten_p ); - -#ifdef ISM_NON_DIEGETIC_PAN - if ( hIsmMetaData[nS]->non_diegetic_flag ) - { - Pos[0] = 0; - Pos[1] = hIsmMetaData[nS]->azimuth / 90.f; - Pos[2] = 0; - TDREND_MIX_SRC_SetPos( hBinRendererTd, nS, Pos ); - TDREND_MIX_SRC_SetPlayState( hBinRendererTd, nS, TDREND_PLAYSTATUS_PLAYING_NON_DIEGETIC ); - } - else - { - TDREND_MIX_SRC_SetPlayState( hBinRendererTd, nS, TDREND_PLAYSTATUS_PLAYING ); - } -#else - TDREND_MIX_SRC_SetPlayState( hBinRendererTd, nS, TDREND_PLAYSTATUS_PLAYING ); -#endif - TDREND_MIX_SRC_SetDir( hBinRendererTd, nS, Dir ); - } - } - - return; -} - - -/*---------------------------------------------------------------------* - * TDREND_Update_listener_orientation() - * - * Update listener orientation (s) - *---------------------------------------------------------------------*/ - -void TDREND_Update_listener_orientation( - BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD Renderer handle */ - const int16_t headRotEnabled, /* i : Headrotation flag */ - const IVAS_QUATERNION *headPosition, /* i : Listener orientation */ - const IVAS_VECTOR3 *Pos /* i : Listener Position */ -) -{ - float FrontVec[3]; - float UpVec[3]; - float Rmat[3][3]; - float Pos_p[3]; - - if ( headRotEnabled ) - { - /* Obtain head rotation matrix */ - QuatToRotMat( *headPosition, Rmat ); - /* Apply rotation matrix to looking vector [1;0;0] */ - FrontVec[0] = Rmat[0][0]; - FrontVec[1] = Rmat[0][1]; - FrontVec[2] = Rmat[0][2]; - /* Apply rotation matrix to up vector [0;0;1] */ - UpVec[0] = Rmat[2][0]; - UpVec[1] = Rmat[2][1]; - UpVec[2] = Rmat[2][2]; - /* Input position */ - Pos_p[0] = ( *Pos ).x; - Pos_p[1] = ( *Pos ).y; - Pos_p[2] = ( *Pos ).z; - } - else - { - /* Oriented with looking vector along the x axis */ - FrontVec[0] = 1.0f; - FrontVec[1] = 0.0f; - FrontVec[2] = 0.0f; - /* Oriented with up vector along the z axis */ - UpVec[0] = 0.0f; - UpVec[1] = 0.0f; - UpVec[2] = 1.0f; - /* Listener at the origin */ - Pos_p[0] = 0.0f; - Pos_p[1] = 0.0f; - Pos_p[2] = 0.0f; - } - - /* Set the listener position and orientation:*/ - TDREND_MIX_LIST_SetPos( hBinRendererTd, Pos_p ); - TDREND_MIX_LIST_SetOrient( hBinRendererTd, FrontVec, UpVec ); - - return; -} - - -/*---------------------------------------------------------------------* - * ivas_td_binaural_open_ext() - * - * - *---------------------------------------------------------------------*/ - -ivas_error ivas_td_binaural_open_ext( - TDREND_WRAPPER *pTDRend, - IVAS_REND_AudioConfig inConfig, - RENDER_CONFIG_DATA *hRendCfg, /* i : Renderer configuration */ - LSSETUP_CUSTOM_STRUCT *customLsInput, - const int32_t outFs ) -{ - int16_t nchan_transport; - AUDIO_CONFIG transport_config; - IVAS_FORMAT ivas_format; - IVAS_OUTPUT_SETUP hTransSetup; - ivas_error error; - float *directivity = NULL; - - if ( inConfig != IVAS_REND_AUDIO_CONFIG_LS_CUSTOM ) - { - if ( ( error = getAudioConfigNumChannels( inConfig, &nchan_transport ) ) != IVAS_ERR_OK ) - { - return error; - } - } - else - { - nchan_transport = customLsInput->num_spk; - } - - transport_config = getIvasAudioConfigFromRendAudioConfig( inConfig ); - ivas_format = ( getAudioConfigType( inConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) ? MC_FORMAT : ISM_FORMAT; - hTransSetup.ls_azimuth = customLsInput->ls_azimuth; - hTransSetup.ls_elevation = customLsInput->ls_elevation; - - if ( NULL != hRendCfg ) - { - directivity = hRendCfg->directivity; - } - - return ivas_td_binaural_open_unwrap( &pTDRend->hHrtfTD, outFs, nchan_transport, ivas_format, transport_config, directivity, hTransSetup, &pTDRend->hBinRendererTd, &pTDRend->binaural_latency_ns ); -} - - -/*---------------------------------------------------------------------* - * ivas_td_binaural_renderer_ext() - * - * Receives the current frames for the object streams, updates metadata - * and renders the current frame. - *---------------------------------------------------------------------*/ - -ivas_error ivas_td_binaural_renderer_ext( - const TDREND_WRAPPER *pTDRend, /* i : TD Renderer wrapper structure */ - const IVAS_REND_AudioConfig inConfig, /* i : Input audio configuration */ - const LSSETUP_CUSTOM_STRUCT *customLsInput, /* i : Input custom loudspeaker layout */ - const IVAS_REND_HeadRotData *headRotData, /* i : Input head positions */ - const IVAS_REND_AudioObjectPosition *currentPos, /* i : Object position */ - const REVERB_HANDLE hReverb, /* i : Reverberator handle */ - const int16_t output_frame, /* i : output frame length */ - float output[][L_FRAME48k] /* i/o: SCE channels / Binaural synthesis */ -) -{ - ISM_METADATA_FRAME hIsmMetaDataFrame; - ISM_METADATA_HANDLE hIsmMetaData[1]; - int16_t lfe_idx; - int16_t num_src; - IVAS_FORMAT ivas_format; - IVAS_REND_AudioConfigType inConfigType; - AUDIO_CONFIG transport_config; - ivas_error error; -#ifdef FIX_356_ISM_METADATA_SYNC - int16_t ism_md_subframe_update = 0; -#endif -#ifdef JBM_TSM_ON_TCS - float *p_output[MAX_OUTPUT_CHANNELS]; - int16_t ch; - - for ( ch = 0; ch < MAX_OUTPUT_CHANNELS; ch++ ) - { - p_output[ch] = output[ch]; - } -#endif - - push_wmops( "ivas_td_binaural_renderer_ext" ); - - inConfigType = getAudioConfigType( inConfig ); - lfe_idx = LFE_CHANNEL; - hIsmMetaData[0] = NULL; - - if ( inConfigType == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) - { - ivas_format = MC_FORMAT; - transport_config = getIvasAudioConfigFromRendAudioConfig( inConfig ); - if ( inConfig != IVAS_REND_AUDIO_CONFIG_LS_CUSTOM ) - { - if ( ( error = getAudioConfigNumChannels( inConfig, &num_src ) ) != IVAS_ERR_OK ) - { - return error; - } - } - else - { - lfe_idx = ( customLsInput->num_lfe > 0 ) ? customLsInput->lfe_idx[0] : -1; - num_src = customLsInput->num_spk + customLsInput->num_lfe; - } - } - else - { - ivas_format = ISM_FORMAT; - num_src = 1; - transport_config = AUDIO_CONFIG_ISM1; - hIsmMetaData[0] = &hIsmMetaDataFrame; - hIsmMetaData[0]->azimuth = currentPos->azimuth; - hIsmMetaData[0]->elevation = currentPos->elevation; - hIsmMetaData[0]->yaw = currentPos->yaw; - hIsmMetaData[0]->pitch = currentPos->pitch; - hIsmMetaData[0]->radius = currentPos->radius; -#ifdef ISM_NON_DIEGETIC_PAN - hIsmMetaData[0]->non_diegetic_flag = currentPos->non_diegetic_flag; -#endif - } - -#ifdef JBM_TSM_ON_TCS -#ifdef FIX_356_ISM_METADATA_SYNC - if( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, - ( headRotData != NULL ) ? headRotData->headPositions : NULL, - ( headRotData != NULL ) ? headRotData->Pos : NULL, ism_md_subframe_update, p_output, output_frame ) ) != IVAS_ERR_OK ) -#else - if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, - ( headRotData != NULL ) ? headRotData->headPositions : NULL, - ( headRotData != NULL ) ? headRotData->Pos : NULL, p_output, output_frame ) ) != IVAS_ERR_OK ) -#endif -#else -#ifdef FIX_356_ISM_METADATA_SYNC - if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, - ( headRotData != NULL ) ? headRotData->headPositions : NULL, - ( headRotData != NULL ) ? headRotData->Pos : NULL, ism_md_subframe_update, output, output_frame ) ) != IVAS_ERR_OK ) -#else - if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, - ( headRotData != NULL ) ? headRotData->headPositions : NULL, - ( headRotData != NULL ) ? headRotData->Pos : NULL, output, output_frame ) ) != IVAS_ERR_OK ) -#endif -#endif - { - return error; - } - - pop_wmops(); - - return IVAS_ERR_OK; -} - -/*---------------------------------------------------------------------* - * angles_to_vec() - * - * Convert azimuth and elevation angles to position/orientation vector - *---------------------------------------------------------------------*/ - -static void angles_to_vec( - const float radius, /* i : radius */ - const float azimuth, /* i : Azimuth angle */ - const float elevation, /* i : Elevation angle */ - float *vec /* o : Pos/Dir vector */ -) -{ - vec[0] = radius * cosf( elevation * PI_OVER_180 ) * cosf( azimuth * PI_OVER_180 ); - vec[1] = radius * cosf( elevation * PI_OVER_180 ) * sinf( azimuth * PI_OVER_180 ); - vec[2] = radius * sinf( elevation * PI_OVER_180 ); - - return; -} -- GitLab From c6be522ed0147e383b119482d6170beb1a2ee77f Mon Sep 17 00:00:00 2001 From: Sumeyra Kanik Date: Wed, 17 May 2023 17:55:27 +0200 Subject: [PATCH 15/18] Fix clang format --- lib_rend/ivas_objectRenderer.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib_rend/ivas_objectRenderer.c b/lib_rend/ivas_objectRenderer.c index 3db591d094..edf15aa1d1 100644 --- a/lib_rend/ivas_objectRenderer.c +++ b/lib_rend/ivas_objectRenderer.c @@ -756,9 +756,9 @@ ivas_error ivas_td_binaural_renderer_ext( #ifdef JBM_TSM_ON_TCS #ifdef FIX_356_ISM_METADATA_SYNC - if( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, - ( headRotData != NULL ) ? headRotData->headPositions : NULL, - ( headRotData != NULL ) ? headRotData->Pos : NULL, ism_md_subframe_update, p_output, output_frame ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, + ( headRotData != NULL ) ? headRotData->headPositions : NULL, + ( headRotData != NULL ) ? headRotData->Pos : NULL, ism_md_subframe_update, p_output, output_frame ) ) != IVAS_ERR_OK ) #else if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, ( headRotData != NULL ) ? headRotData->headPositions : NULL, -- GitLab From dafc30fe586ae9b31d76230cd7b71e5e701c4399 Mon Sep 17 00:00:00 2001 From: Sumeyra Kanik Date: Mon, 22 May 2023 09:36:24 +0200 Subject: [PATCH 16/18] Clang Format --- lib_com/ivas_stat_com.h | 12 ++++++------ lib_rend/ivas_objectRenderer.c | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib_com/ivas_stat_com.h b/lib_com/ivas_stat_com.h index d5d1f27103..b4d0c369b3 100644 --- a/lib_com/ivas_stat_com.h +++ b/lib_com/ivas_stat_com.h @@ -59,13 +59,13 @@ typedef struct int16_t ism_metadata_flag; /* flag whether metadata are coded in particular frame of particular object */ int16_t last_ism_metadata_flag; /* last frame ism_metadata_flag */ - float azimuth; /* azimuth value read from the input metadata file */ - float elevation; /* elevation value read from the input metadata file */ - float radius; /* radius value read from the input metadata file */ - float yaw; /* yaw value read from the input metadata file */ - float pitch; /* pitch value read from the input metadata file */ + float azimuth; /* azimuth value read from the input metadata file */ + float elevation; /* elevation value read from the input metadata file */ + float radius; /* radius value read from the input metadata file */ + float yaw; /* yaw value read from the input metadata file */ + float pitch; /* pitch value read from the input metadata file */ - int16_t non_diegetic_flag; /* Non-diegetic (non-headtracked) object flag */ + int16_t non_diegetic_flag; /* Non-diegetic (non-headtracked) object flag */ ISM_METADATA_ANGLE position_angle; /* Angle structs for azimuth and elevation */ ISM_METADATA_ANGLE orientation_angle; /* Angle structs for yaw and pitch */ diff --git a/lib_rend/ivas_objectRenderer.c b/lib_rend/ivas_objectRenderer.c index 8a09814edf..142d35fec3 100644 --- a/lib_rend/ivas_objectRenderer.c +++ b/lib_rend/ivas_objectRenderer.c @@ -492,7 +492,7 @@ void TDREND_Update_object_positions( #ifdef JBM_TSM_ON_TCS float *output[] #else - float output[][L_FRAME48k] /* i/o: SCE/MC channels */ + float output[][L_FRAME48k] /* i/o: SCE/MC channels */ #endif #endif ) -- GitLab From 40a6fe697b660e71776295c9b010bd724f94f693 Mon Sep 17 00:00:00 2001 From: Sumeyra Kanik Date: Mon, 22 May 2023 18:59:13 +0200 Subject: [PATCH 17/18] Fix format issues --- lib_dec/ivas_ism_renderer.c | 6 ------ lib_dec/ivas_sba_rendering_internal.c | 2 -- lib_rend/ivas_objectRenderer.c | 4 ++-- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/lib_dec/ivas_ism_renderer.c b/lib_dec/ivas_ism_renderer.c index 779751d7b6..3b38e2d890 100644 --- a/lib_dec/ivas_ism_renderer.c +++ b/lib_dec/ivas_ism_renderer.c @@ -168,8 +168,6 @@ void ivas_ism_render( { if ( st_ivas->intern_config == AUDIO_CONFIG_STEREO ) { -#ifdef FIX_356_ISM_METADATA_SYNC -#endif ivas_ism_get_stereo_gains( st_ivas->hIsmMetaData[i]->azimuth, st_ivas->hIsmMetaData[i]->elevation, &gains[i][0], @@ -180,15 +178,11 @@ void ivas_ism_render( /* Head rotation: rotate the object positions depending the head's orientation */ if ( st_ivas->hHeadTrackData != NULL && st_ivas->hHeadTrackData->num_quaternions >= 0 && !st_ivas->hIsmMetaData[i]->non_diegetic_flag ) { -#ifdef FIX_356_ISM_METADATA_SYNC -#endif rotateAziEle( st_ivas->hIsmMetaData[i]->azimuth, st_ivas->hIsmMetaData[i]->elevation, &azimuth, &elevation, Rmat, st_ivas->hIntSetup.is_planar_setup ); } else { // TODO tmu review when #215 is resolved -#ifdef FIX_356_ISM_METADATA_SYNC -#endif azimuth = (int16_t) floorf( st_ivas->hIsmMetaData[i]->azimuth + 0.5f ); elevation = (int16_t) floorf( st_ivas->hIsmMetaData[i]->elevation + 0.5f ); diff --git a/lib_dec/ivas_sba_rendering_internal.c b/lib_dec/ivas_sba_rendering_internal.c index ef3c51d234..71a5718436 100644 --- a/lib_dec/ivas_sba_rendering_internal.c +++ b/lib_dec/ivas_sba_rendering_internal.c @@ -334,8 +334,6 @@ void ivas_ism2sba( for ( i = 0; i < nchan_ism; i++ ) { -#ifdef FIX_356_ISM_METADATA_SYNC -#endif // TODO tmu review when #215 is resolved azimuth = (int16_t) floorf( hIsmMetaData[i]->azimuth + 0.5f ); elevation = (int16_t) floorf( hIsmMetaData[i]->elevation + 0.5f ); diff --git a/lib_rend/ivas_objectRenderer.c b/lib_rend/ivas_objectRenderer.c index 142d35fec3..3d929e78c7 100644 --- a/lib_rend/ivas_objectRenderer.c +++ b/lib_rend/ivas_objectRenderer.c @@ -691,7 +691,7 @@ ivas_error ivas_td_binaural_renderer_ext( AUDIO_CONFIG transport_config; ivas_error error; #ifdef FIX_356_ISM_METADATA_SYNC - int16_t ism_md_subframe_update = 0; + int16_t ism_md_subframe_update_ext = 0; #endif #ifdef JBM_TSM_ON_TCS float *p_output[MAX_OUTPUT_CHANNELS]; @@ -744,7 +744,7 @@ ivas_error ivas_td_binaural_renderer_ext( #ifdef FIX_356_ISM_METADATA_SYNC if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, ( headRotData != NULL ) ? headRotData->headPositions : NULL, - ( headRotData != NULL ) ? headRotData->Pos : NULL, ism_md_subframe_update, p_output, output_frame ) ) != IVAS_ERR_OK ) + ( headRotData != NULL ) ? headRotData->Pos : NULL, ism_md_subframe_update_ext, p_output, output_frame ) ) != IVAS_ERR_OK ) #else if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, ( headRotData != NULL ) ? headRotData->headPositions : NULL, -- GitLab From 09958e8de17f73d22f3645507bae7c1405b20411 Mon Sep 17 00:00:00 2001 From: Sumeyra Kanik Date: Tue, 23 May 2023 11:29:22 +0200 Subject: [PATCH 18/18] Merge from main, recent updates --- lib_dec/ivas_objectRenderer_internal.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib_dec/ivas_objectRenderer_internal.c b/lib_dec/ivas_objectRenderer_internal.c index 3f7d05a07a..081b33e55c 100644 --- a/lib_dec/ivas_objectRenderer_internal.c +++ b/lib_dec/ivas_objectRenderer_internal.c @@ -113,7 +113,7 @@ void ObjRenderIVASSubframe( float *tc_local[MAX_TRANSPORT_CHANNELS]; int16_t ch, slot_size, slots_to_render, output_frame; #ifdef FIX_356_ISM_METADATA_SYNC - int16_t ism_md_subframe_update = 0; /* Number of subframes to delay metadata to sync with audio */ + int16_t ism_md_subframe_update_jbm = 0; /* Number of subframes to delay metadata to sync with audio */ #endif for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) { @@ -163,7 +163,7 @@ void ObjRenderIVASSubframe( /* Render subframe */ #ifdef FIX_356_ISM_METADATA_SYNC - TDREND_GetMix( st_ivas->hBinRendererTd, output_f_local, output_frame, 0, ism_md_subframe_update ); + TDREND_GetMix( st_ivas->hBinRendererTd, output_f_local, output_frame, 0, ism_md_subframe_update_jbm ); #else TDREND_GetMix( st_ivas->hBinRendererTd, output_f_local, output_frame, 0 ); #endif -- GitLab