From 424356fd0b8c715c3d3321a834d30fcf5e3ea55c Mon Sep 17 00:00:00 2001 From: Lauros Pajunen Date: Thu, 8 Feb 2024 09:55:55 +0200 Subject: [PATCH 1/7] Enable object editing interface with a callback --- apps/decoder.c | 58 ++++++++++- lib_com/options.h | 3 + lib_dec/ivas_ism_renderer.c | 26 +++++ lib_dec/lib_dec.c | 103 +++++++++++++++++++ lib_dec/lib_dec.h | 8 ++ lib_rend/ivas_dirac_dec_binaural_functions.c | 26 +++++ 6 files changed, 223 insertions(+), 1 deletion(-) diff --git a/apps/decoder.c b/apps/decoder.c index 00efa03735..0c1dc1ab5d 100644 --- a/apps/decoder.c +++ b/apps/decoder.c @@ -144,7 +144,9 @@ typedef struct uint16_t acousticEnvironmentId; int16_t Opt_dpid_on; uint16_t directivityPatternId[IVAS_MAX_NUM_OBJECTS]; - +#ifdef ENABLE_OBJECT_EDITING + bool objEditEnabled; +#endif } DecArguments; @@ -161,6 +163,10 @@ static ivas_error decodeG192( DecArguments arg, BS_READER_HANDLE hBsReader, RotF #endif static ivas_error decodeVoIP( DecArguments arg, BS_READER_HANDLE hBsReader, RotFileReader *headRotReader, RotFileReader *externalOrientationFileReader, RotFileReader *refRotReader, Vector3PairFileReader *referenceVectorReader, IVAS_DEC_HANDLE hIvasDec ); +#ifdef ENABLE_OBJECT_EDITING +static void exampleObjEditFun(IVAS_ISM_METADATA *obj_metas[IVAS_MAX_NUM_OBJECTS], uint16_t n_objs, bool* obj_modified_flag); +#endif + #ifdef DEBUGGING static ivas_error printBitstreamInfoVoip( DecArguments arg, BS_READER_HANDLE hBsReader, IVAS_DEC_HANDLE hIvasDec ); static int16_t app_own_random( int16_t *seed ); @@ -1113,6 +1119,9 @@ static bool parseCmdlIVAS_dec( { arg->directivityPatternId[i] = 65535; } +#ifdef ENABLE_OBJECT_EDITING + arg->objEditEnabled = false; +#endif /*-----------------------------------------------------------------* * Initialization @@ -1539,6 +1548,13 @@ static bool parseCmdlIVAS_dec( i += tmp; } +#ifdef ENABLE_OBJECT_EDITING + else if ( strcmp( argv_to_upper, "-OBJ_EDIT" ) == 0 ) + { + arg->objEditEnabled = true; + i++; + } +#endif /*-----------------------------------------------------------------* * Option not recognized @@ -1740,6 +1756,9 @@ static void usage_dec( void ) fprintf( stdout, " Currently, all values default to level 3 (full functionality).\n" ); fprintf( stdout, "-q : Quiet mode, no frame counter\n" ); fprintf( stdout, " default is deactivated\n" ); +#ifdef ENABLE_OBJECT_EDITING + fprintf( stdout, "-obj_edit : enable object editing\n" ); +#endif #ifdef DEBUG_MODE_INFO #ifdef DEBUG_MODE_INFO_TWEAK fprintf( stdout, "-info : specify subfolder name for debug output\n" ); @@ -1937,6 +1956,16 @@ static ivas_error initOnFirstGoodFrame( free( zeroBuf ); +#ifdef ENABLE_OBJECT_EDITING + if ( arg.objEditEnabled ) + { + if ( ( error = IVAS_DEC_SetObjectEditCallback( hIvasDec, exampleObjEditFun ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError in IVAS_DEC_SetObjectEditCallback: %s\nContinuing without object editing.\n", IVAS_DEC_GetErrorMessage( error ) ); + } + } +#endif + /* Open other output files if EXT output config - now details about ISM or MASA are known */ if ( arg.outputConfig == IVAS_AUDIO_CONFIG_EXTERNAL ) { @@ -3462,4 +3491,31 @@ static IVAS_DEC_FORCED_REND_MODE parseForcedRendModeDec( } #endif +#ifdef ENABLE_OBJECT_EDITING +/*---------------------------------------------------------------------* + * exampleObjEditFun() + * + * An example object editing callback function + *---------------------------------------------------------------------*/ +static void exampleObjEditFun(IVAS_ISM_METADATA *obj_metas[IVAS_MAX_NUM_OBJECTS], uint16_t n_objs, bool* obj_modified_flag) +{ + uint16_t obj_idx; + + for ( obj_idx = 0; obj_idx < n_objs; obj_idx++ ) + { + /* limit azimuth to the front */ + if ( obj_metas[obj_idx]->azimuth > 90.0f ) + { + obj_metas[obj_idx]->azimuth = 90.0f; + obj_modified_flag[obj_idx] = true; + } + else if ( obj_metas[obj_idx]->azimuth < -90.0f ) + { + obj_metas[obj_idx]->azimuth = -90.0f; + obj_modified_flag[obj_idx] = true; + } + } +} +#endif + #undef WMC_TOOL_SKIP diff --git a/lib_com/options.h b/lib_com/options.h index e1eaa1221c..e6bd10777b 100755 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -154,6 +154,9 @@ #define FIX_963_USAN_ERROR /* Eri: Issue 963: USAN error in Stereo CNG, division by zero */ #define FIX_971_LOG2_IDX_GAIN_0 /* VA: prevent -Inf due to log2(ratio==0) */ +#define ENABLE_OBJECT_EDITING /* Nokia: development switch for object editing interface */ + + /* #################### End BE switches ################################## */ /* #################### Start NON-BE switches ############################ */ diff --git a/lib_dec/ivas_ism_renderer.c b/lib_dec/ivas_ism_renderer.c index 9fcb8f9f43..5a67bb0114 100644 --- a/lib_dec/ivas_ism_renderer.c +++ b/lib_dec/ivas_ism_renderer.c @@ -549,13 +549,39 @@ void ivas_omasa_separate_object_render_jbm( if ( single_separated ) { +#ifdef ENABLE_OBJECT_EDITING + if ( st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ && st_ivas->hMasaIsmData->ism_is_edited[st_ivas->hMasaIsmData->idx_separated_ism] ) + { + azimuth = st_ivas->hMasaIsmData->azimuth_ism_edited[st_ivas->hMasaIsmData->idx_separated_ism]; + elevation = st_ivas->hMasaIsmData->elevation_ism_edited[st_ivas->hMasaIsmData->idx_separated_ism]; + } + else + { + azimuth = st_ivas->hMasaIsmData->azimuth_separated_ism[md_idx]; + elevation = st_ivas->hMasaIsmData->elevation_separated_ism[md_idx]; + } +#else azimuth = st_ivas->hMasaIsmData->azimuth_separated_ism[md_idx]; elevation = st_ivas->hMasaIsmData->elevation_separated_ism[md_idx]; +#endif } else { +#ifdef ENABLE_OBJECT_EDITING + if ( st_ivas->hMasaIsmData->ism_is_edited[obj] ) + { + azimuth = st_ivas->hMasaIsmData->azimuth_ism_edited[obj]; + elevation = st_ivas->hMasaIsmData->elevation_ism_edited[obj]; + } + else + { + azimuth = st_ivas->hMasaIsmData->azimuth_ism[obj][md_idx]; + elevation = st_ivas->hMasaIsmData->elevation_ism[obj][md_idx]; + } +#else azimuth = st_ivas->hMasaIsmData->azimuth_ism[obj][md_idx]; elevation = st_ivas->hMasaIsmData->elevation_ism[obj][md_idx]; +#endif } if ( st_ivas->hOutSetup.is_planar_setup ) diff --git a/lib_dec/lib_dec.c b/lib_dec/lib_dec.c index 564543a800..3d58a2dddb 100644 --- a/lib_dec/lib_dec.c +++ b/lib_dec/lib_dec.c @@ -91,6 +91,9 @@ struct IVAS_DEC int16_t sdp_hf_only; /* RTP payload format parameter: only Header-Full format without zero padding for size collision avoidance */ int16_t prev_ft_speech; /* RXDTX handler: previous frametype flag for G.192 format AMRWB SID_FIRST detection */ int16_t CNG; /* RXDTX handler: CNG=1, nonCNG=0 */ +#ifdef ENABLE_OBJECT_EDITING + IvasObjEditCallbackFn hObjectEditCallback; /* object editing callback function */ +#endif }; @@ -180,6 +183,9 @@ ivas_error IVAS_DEC_Open( hIvasDec->amrwb_rfc4867_flag = -1; hIvasDec->prev_ft_speech = 1; /* RXDTX handler previous frametype flag for G.192 format AMRWB SID_FIRST detection */ hIvasDec->CNG = 0; /* RXDTX handler CNG = 1, no CNG = 0*/ +#ifdef ENABLE_OBJECT_EDITING + hIvasDec->hObjectEditCallback = NULL; +#endif /*-----------------------------------------------------------------* * Initialize IVAS-codec decoder state @@ -971,6 +977,86 @@ ivas_error IVAS_DEC_GetSamples( nSamplesTcsScaled = hIvasDec->nSamplesFrame; } +#ifdef ENABLE_OBJECT_EDITING + if ( ( hIvasDec->st_ivas->ivas_format == ISM_FORMAT || + ( hIvasDec->st_ivas->ivas_format == SBA_ISM_FORMAT && hIvasDec->st_ivas->ism_mode == ISM_SBA_MODE_DISC ) || + ( hIvasDec->st_ivas->ivas_format == MASA_ISM_FORMAT && ( hIvasDec->st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ || hIvasDec->st_ivas->ism_mode == ISM_MASA_MODE_DISC ) ) || + ( hIvasDec->st_ivas->ivas_format == MASA_FORMAT && hIvasDec->st_ivas->nchan_ism > 0 ) ) && hIvasDec->hObjectEditCallback != NULL ) + { + uint16_t ism_idx; + + /* copy object metadata into a simplified structure for providing outside */ + IVAS_ISM_METADATA IsmMetadata[IVAS_MAX_NUM_OBJECTS]; + IVAS_ISM_METADATA* hIsmMetadata[IVAS_MAX_NUM_OBJECTS]; + bool mod_flag[IVAS_MAX_NUM_OBJECTS]; + + for ( ism_idx = 0; ism_idx < IVAS_MAX_NUM_OBJECTS; ism_idx++ ) + { + mod_flag[ism_idx] = false; + } + for ( ism_idx = 0; ism_idx < hIvasDec->st_ivas->nchan_ism; ism_idx++ ) + { + hIsmMetadata[ism_idx] = &(IsmMetadata[ism_idx]); + if ( ( error = IVAS_DEC_GetObjectMetadata( hIvasDec, hIsmMetadata[ism_idx], 0, ism_idx ) ) != IVAS_ERR_OK ) + { + return error; + } + } + + /* execute callback modifying the metadata */ + hIvasDec->hObjectEditCallback(hIsmMetadata, hIvasDec->st_ivas->nchan_ism, mod_flag ); + + /* copy edited data from callback */ + for ( ism_idx = 0; ism_idx < hIvasDec->st_ivas->nchan_ism; ism_idx++ ) + { + if ( mod_flag[ism_idx] ) + { + IVAS_ISM_METADATA* srcIsmMeta = hIsmMetadata[ism_idx]; + ISM_METADATA_HANDLE destIsmMeta = hIvasDec->st_ivas->hIsmMetaData[ism_idx]; + + destIsmMeta->azimuth = srcIsmMeta->azimuth; + destIsmMeta->elevation = srcIsmMeta->elevation; + destIsmMeta->radius = srcIsmMeta->radius; + destIsmMeta->yaw = srcIsmMeta->yaw; + destIsmMeta->pitch = srcIsmMeta->pitch; + destIsmMeta->non_diegetic_flag = srcIsmMeta->non_diegetic_flag; + + if ( hIvasDec->st_ivas->hMasaIsmData != NULL ) + { + if ( srcIsmMeta->azimuth > 0.0f ) + { + hIvasDec->st_ivas->hMasaIsmData->azimuth_ism_edited[ism_idx] = (int16_t) ( srcIsmMeta->azimuth + 0.5f ); + } + else + { + hIvasDec->st_ivas->hMasaIsmData->azimuth_ism_edited[ism_idx] = (int16_t) ( srcIsmMeta->azimuth - 0.5f ); + } + + if ( srcIsmMeta->elevation > 0.0f ) + { + hIvasDec->st_ivas->hMasaIsmData->elevation_ism_edited[ism_idx] = (int16_t) ( srcIsmMeta->elevation + 0.5f ); + } + else + { + hIvasDec->st_ivas->hMasaIsmData->elevation_ism_edited[ism_idx] = (int16_t) ( srcIsmMeta->elevation - 0.5f ); + } + hIvasDec->st_ivas->hMasaIsmData->ism_is_edited[ism_idx] = 1u; + hIvasDec->st_ivas->hMasaIsmData->objectsMoved = 1u; + } + } + else + { + if ( hIvasDec->st_ivas->hMasaIsmData != NULL ) + { + /* reset modification flag */ + hIvasDec->st_ivas->hMasaIsmData->ism_is_edited[ism_idx] = 0u; + } + + } + } + } +#endif + /* Feed decoded transport channels samples to the renderer */ if ( ( error = IVAS_DEC_RendererFeedTcSamples( hIvasDec, nSamplesTcsScaled, &nResidualSamples, hIvasDec->apaExecBuffer ) ) != IVAS_ERR_OK ) { @@ -2249,6 +2335,23 @@ ivas_error IVAS_DEC_GetPcmFrameSize( return IVAS_ERR_OK; } +#ifdef ENABLE_OBJECT_EDITING +ivas_error IVAS_DEC_SetObjectEditCallback( + IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ + IvasObjEditCallbackFn callbackFn /* i : object editing callback function */ +) +{ + if ( hIvasDec->st_ivas->ivas_format == ISM_FORMAT || hIvasDec->st_ivas->ivas_format == SBA_ISM_FORMAT || hIvasDec->st_ivas->ivas_format == MASA_ISM_FORMAT ) + { + hIvasDec->hObjectEditCallback = callbackFn; + return IVAS_ERR_OK; + } + else + { + return IVAS_ERR_WRONG_MODE; + } +} +#endif /*---------------------------------------------------------------------* * isSidFrame( ) diff --git a/lib_dec/lib_dec.h b/lib_dec/lib_dec.h index eafd391b14..47b1bf5586 100644 --- a/lib_dec/lib_dec.h +++ b/lib_dec/lib_dec.h @@ -500,6 +500,14 @@ void IVAS_DEC_PrintConfig( const bool voipMode ); +#ifdef ENABLE_OBJECT_EDITING +typedef void (*IvasObjEditCallbackFn)(IVAS_ISM_METADATA*[IVAS_MAX_NUM_OBJECTS], uint16_t, bool* ); +ivas_error IVAS_DEC_SetObjectEditCallback( + IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ + IvasObjEditCallbackFn callbackFn /* i : object editing callback function */ +); +#endif + #ifdef DEBUGGING void IVAS_DEC_PrintConfigWithBitstream( IVAS_DEC_HANDLE hIvasDec, /* i : IVAS decoder handle */ diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index 0d6d20e7ed..a37d55ea8e 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -2174,13 +2174,39 @@ static void ivas_dirac_dec_binaural_determine_processing_matrices( { if ( ism_mode == ISM_MASA_MODE_DISC ) { +#ifdef ENABLE_OBJECT_EDITING + if ( hMasaIsmData->ism_is_edited[chB] ) + { + aziDeg = hMasaIsmData->azimuth_ism_edited[chB]; + eleDeg = hMasaIsmData->elevation_ism_edited[chB]; + } + else + { + aziDeg = hMasaIsmData->azimuth_ism[chB][dirac_read_idx]; + eleDeg = hMasaIsmData->elevation_ism[chB][dirac_read_idx]; + } +#else aziDeg = hMasaIsmData->azimuth_ism[chB][dirac_read_idx]; eleDeg = hMasaIsmData->elevation_ism[chB][dirac_read_idx]; +#endif } else { +#ifdef ENABLE_OBJECT_EDITING + if ( ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ && hMasaIsmData->ism_is_edited[hMasaIsmData->idx_separated_ism] ) + { + aziDeg = hMasaIsmData->azimuth_ism_edited[hMasaIsmData->idx_separated_ism]; + eleDeg = hMasaIsmData->elevation_ism_edited[hMasaIsmData->idx_separated_ism]; + } + else + { + aziDeg = hMasaIsmData->azimuth_separated_ism[dirac_read_idx]; + eleDeg = hMasaIsmData->elevation_separated_ism[dirac_read_idx]; + } +#else aziDeg = hMasaIsmData->azimuth_separated_ism[dirac_read_idx]; eleDeg = hMasaIsmData->elevation_separated_ism[dirac_read_idx]; +#endif instantChange = 1; } } -- GitLab From 3a1052afa35bc240fffa0c1c8819caf68273d53f Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Tue, 16 Apr 2024 12:22:53 +0300 Subject: [PATCH 2/7] Provides gain editing objects and MASA in OMASA rendering. Refactors direction editing and parametric stereo rendering code to avoid unnecessary complexity. Changes ISM metadata reservation to save memory and fully support rate switching together with object editing. --- apps/decoder.c | 20 +- lib_com/ivas_cnst.h | 3 + lib_com/ivas_prot.h | 10 + lib_com/options.h | 4 +- lib_dec/ivas_dirac_dec.c | 7 + lib_dec/ivas_init_dec.c | 12 + lib_dec/ivas_ism_renderer.c | 4 +- lib_dec/ivas_jbm_dec.c | 27 + lib_dec/ivas_omasa_dec.c | 82 ++- lib_dec/ivas_stat_dec.h | 21 + lib_dec/lib_dec.c | 43 +- lib_dec/lib_dec.h | 2 +- lib_rend/ivas_dirac_dec_binaural_functions.c | 554 ++++++++++++++++++- lib_rend/ivas_dirac_output_synthesis_dec.c | 14 + 14 files changed, 769 insertions(+), 34 deletions(-) diff --git a/apps/decoder.c b/apps/decoder.c index 049e3682a0..592606d210 100644 --- a/apps/decoder.c +++ b/apps/decoder.c @@ -164,7 +164,7 @@ static ivas_error decodeG192( DecArguments arg, BS_READER_HANDLE hBsReader, RotF static ivas_error decodeVoIP( DecArguments arg, BS_READER_HANDLE hBsReader, RotFileReader *headRotReader, RotFileReader *externalOrientationFileReader, RotFileReader *refRotReader, Vector3PairFileReader *referenceVectorReader, IVAS_DEC_HANDLE hIvasDec ); #ifdef ENABLE_OBJECT_EDITING -static void exampleObjEditFun(IVAS_ISM_METADATA *obj_metas[IVAS_MAX_NUM_OBJECTS], uint16_t n_objs, bool* obj_modified_flag); +static void exampleObjEditFun( IVAS_ISM_METADATA *obj_metas[IVAS_MAX_NUM_OBJECTS], uint16_t n_objs, bool *obj_modified_flag, float *bed_gain ); #endif #ifdef DEBUGGING @@ -3588,24 +3588,18 @@ static IVAS_DEC_FORCED_REND_MODE parseForcedRendModeDec( * * An example object editing callback function *---------------------------------------------------------------------*/ -static void exampleObjEditFun(IVAS_ISM_METADATA *obj_metas[IVAS_MAX_NUM_OBJECTS], uint16_t n_objs, bool* obj_modified_flag) +static void exampleObjEditFun( IVAS_ISM_METADATA *obj_metas[IVAS_MAX_NUM_OBJECTS], uint16_t n_objs, bool *obj_modified_flag, float *bed_gain ) { uint16_t obj_idx; for ( obj_idx = 0; obj_idx < n_objs; obj_idx++ ) { - /* limit azimuth to the front */ - if ( obj_metas[obj_idx]->azimuth > 90.0f ) - { - obj_metas[obj_idx]->azimuth = 90.0f; - obj_modified_flag[obj_idx] = true; - } - else if ( obj_metas[obj_idx]->azimuth < -90.0f ) - { - obj_metas[obj_idx]->azimuth = -90.0f; - obj_modified_flag[obj_idx] = true; - } + obj_metas[obj_idx]->azimuth = (float) obj_idx * 19.0f; + obj_metas[obj_idx]->gainFactor = 1.5f; + obj_modified_flag[obj_idx] = true; } + + bed_gain[0] = 0.7f; } #endif diff --git a/lib_com/ivas_cnst.h b/lib_com/ivas_cnst.h index cd7a38580b..a042752ba7 100755 --- a/lib_com/ivas_cnst.h +++ b/lib_com/ivas_cnst.h @@ -1222,6 +1222,9 @@ enum #define MASA_MAXIMUM_TWO_DIR_BANDS 24 #define NBITS_HR_COH 4 #define OMASA_TDREND_MATCHING_GAIN 0.7943f +#ifdef ENABLE_OBJECT_EDITING +#define OMASA_GAIN_EDIT_THR 0.06f /* OMASA gain change threshold */ +#endif #define MASA_JBM_RINGBUFFER_FRAMES 3 diff --git a/lib_com/ivas_prot.h b/lib_com/ivas_prot.h index 757db691e9..49dc1c34ef 100755 --- a/lib_com/ivas_prot.h +++ b/lib_com/ivas_prot.h @@ -5782,6 +5782,15 @@ void ivas_omasa_dirac_rend_jbm( float *output_f[] /* o : rendered time signal */ ); +#ifdef ENABLE_OBJECT_GAIN_EDITING +void ivas_omasa_preProcessStereoTransportsForEditedObjects( + Decoder_Struct *st_ivas, + float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + const int16_t nBins, + const int16_t subframe +); +#else void ivas_omasa_preProcessStereoTransportsForMovedObjects( Decoder_Struct *st_ivas, float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], @@ -5789,6 +5798,7 @@ void ivas_omasa_preProcessStereoTransportsForMovedObjects( const int16_t nBins, const int16_t subframe ); +#endif ivas_error ivas_omasa_separate_object_renderer_open( Decoder_Struct *st_ivas /* i/o: IVAS decoder structure */ diff --git a/lib_com/options.h b/lib_com/options.h index 33226df5df..afe883516a 100644 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -161,7 +161,9 @@ #define ENABLE_OBJECT_EDITING /* Nokia: development switch for object editing interface */ - +#define ENABLE_OBJECT_GAIN_EDITING /* Nokia: development switch for object gain editing */ +#define STEREO_GAIN_CACHE /* Nokia: development switch for stereo gain cache optimization */ +#define OMASA_ISM_META_OPT /* Nokia: development switch OMASA ISM metadata allocation optimization */ /* #################### End BE switches ################################## */ diff --git a/lib_dec/ivas_dirac_dec.c b/lib_dec/ivas_dirac_dec.c index c8d52aa67e..d26a6132b1 100644 --- a/lib_dec/ivas_dirac_dec.c +++ b/lib_dec/ivas_dirac_dec.c @@ -1869,10 +1869,17 @@ void ivas_dirac_dec_render_sf( } } +#ifdef ENABLE_OBJECT_GAIN_EDITING + if ( st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ ) + { + ivas_omasa_preProcessStereoTransportsForEditedObjects( st_ivas, Cldfb_RealBuffer_Temp, Cldfb_ImagBuffer_Temp, hSpatParamRendCom->num_freq_bands, subframe_idx ); + } +#else if ( st_ivas->ism_mode != ISM_MASA_MODE_DISC && st_ivas->ism_mode != ISM_MASA_MODE_MASA_ONE_OBJ ) { ivas_omasa_preProcessStereoTransportsForMovedObjects( st_ivas, Cldfb_RealBuffer_Temp, Cldfb_ImagBuffer_Temp, hSpatParamRendCom->num_freq_bands, subframe_idx ); } +#endif } for ( slot_idx = 0; slot_idx < hSpatParamRendCom->subframe_nbslots[subframe_idx]; slot_idx++ ) diff --git a/lib_dec/ivas_init_dec.c b/lib_dec/ivas_init_dec.c index 7858a2d80c..604b3505ef 100644 --- a/lib_dec/ivas_init_dec.c +++ b/lib_dec/ivas_init_dec.c @@ -1670,10 +1670,19 @@ ivas_error ivas_init_decoder( reset_indices_dec( st_ivas->hSCE[0]->hCoreCoder[0] ); +#ifdef OMASA_ISM_META_OPT + if ( ( error = ivas_ism_metadata_dec_create( st_ivas, st_ivas->nchan_ism, NULL ) ) != IVAS_ERR_OK ) + { + return error; + } + ivas_ism_metadata_close( st_ivas->hIsmMetaData, st_ivas->nchan_ism ); /* close extra structs */ +#else if ( ( error = ivas_ism_metadata_dec_create( st_ivas, 1, NULL ) ) != IVAS_ERR_OK ) { return error; } + +#endif } else if ( st_ivas->ism_mode == ISM_MASA_MODE_DISC ) { @@ -1697,6 +1706,9 @@ ivas_error ivas_init_decoder( { return error; } +#ifdef OMASA_ISM_META_OPT + ivas_ism_metadata_close( st_ivas->hIsmMetaData, st_ivas->nchan_ism ); /* close extra structs */ +#endif } if ( ( error = ivas_masa_dec_open( st_ivas ) ) != IVAS_ERR_OK ) diff --git a/lib_dec/ivas_ism_renderer.c b/lib_dec/ivas_ism_renderer.c index 1a35999c36..ad2efa1e3a 100644 --- a/lib_dec/ivas_ism_renderer.c +++ b/lib_dec/ivas_ism_renderer.c @@ -544,7 +544,7 @@ void ivas_omasa_separate_object_render_jbm( if ( single_separated ) { #ifdef ENABLE_OBJECT_EDITING - if ( st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ && st_ivas->hMasaIsmData->ism_is_edited[st_ivas->hMasaIsmData->idx_separated_ism] ) + if ( st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ && st_ivas->hMasaIsmData->ism_dir_is_edited[st_ivas->hMasaIsmData->idx_separated_ism] ) { azimuth = st_ivas->hMasaIsmData->azimuth_ism_edited[st_ivas->hMasaIsmData->idx_separated_ism]; elevation = st_ivas->hMasaIsmData->elevation_ism_edited[st_ivas->hMasaIsmData->idx_separated_ism]; @@ -562,7 +562,7 @@ void ivas_omasa_separate_object_render_jbm( else { #ifdef ENABLE_OBJECT_EDITING - if ( st_ivas->hMasaIsmData->ism_is_edited[obj] ) + if ( st_ivas->hMasaIsmData->ism_dir_is_edited[obj] ) { azimuth = st_ivas->hMasaIsmData->azimuth_ism_edited[obj]; elevation = st_ivas->hMasaIsmData->elevation_ism_edited[obj]; diff --git a/lib_dec/ivas_jbm_dec.c b/lib_dec/ivas_jbm_dec.c index e6a2ee3194..fc0f9e76d0 100644 --- a/lib_dec/ivas_jbm_dec.c +++ b/lib_dec/ivas_jbm_dec.c @@ -949,10 +949,37 @@ void ivas_jbm_dec_feed_tc_to_renderer( { if ( st_ivas->ism_mode == ISM_MASA_MODE_DISC && st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC ) { +#ifdef ENABLE_OBJECT_GAIN_EDITING + float gainIsm; + gainIsm = OMASA_TDREND_MATCHING_GAIN; + + if ( st_ivas->hMasaIsmData->ism_gain_is_edited[n] ) + { + gainIsm *= st_ivas->hMasaIsmData->gain_ism_edited[n]; + } + + v_multc( st_ivas->hTcBuffer->tc[CPE_CHANNELS + n], gainIsm, st_ivas->hTcBuffer->tc[CPE_CHANNELS + n], st_ivas->hTcBuffer->n_samples_available ); +#else v_multc( st_ivas->hTcBuffer->tc[CPE_CHANNELS + n], OMASA_TDREND_MATCHING_GAIN, st_ivas->hTcBuffer->tc[CPE_CHANNELS + n], st_ivas->hTcBuffer->n_samples_available ); +#endif } delay_signal( st_ivas->hTcBuffer->tc[CPE_CHANNELS + n], st_ivas->hTcBuffer->n_samples_available, st_ivas->hMasaIsmData->delayBuffer[n], st_ivas->hMasaIsmData->delayBuffer_size ); } +#ifdef ENABLE_OBJECT_GAIN_EDITING + if ( st_ivas->ism_mode == ISM_MASA_MODE_DISC && st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC ) + { + int16_t tcBufferSize; + tcBufferSize = st_ivas->hSpatParamRendCom->num_slots * st_ivas->hSpatParamRendCom->slot_size; + + if ( st_ivas->hMasaIsmData->masa_gain_is_edited ) + { + for ( int16_t ch = 0; ch < 2; ch++ ) + { + v_multc( st_ivas->hTcBuffer->tc[ch], st_ivas->hMasaIsmData->gain_masa_edited, st_ivas->hTcBuffer->tc[ch], tcBufferSize ); + } + } + } +#endif } } } diff --git a/lib_dec/ivas_omasa_dec.c b/lib_dec/ivas_omasa_dec.c index b1a0a0e9ee..ed1c3d858a 100644 --- a/lib_dec/ivas_omasa_dec.c +++ b/lib_dec/ivas_omasa_dec.c @@ -60,7 +60,11 @@ ivas_error ivas_omasa_data_open( ) { MASA_ISM_DATA_HANDLE hMasaIsmData; +#ifdef ENABLE_OBJECT_GAIN_EDITING + int16_t ch; +#else int16_t ch, bin; +#endif int16_t sf, obj_idx; if ( ( hMasaIsmData = (MASA_ISM_DATA_HANDLE) malloc( sizeof( MASA_ISM_DATA ) ) ) == NULL ) @@ -68,6 +72,21 @@ ivas_error ivas_omasa_data_open( return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for MASA ISM data\n" ) ); } +#ifdef ENABLE_OBJECT_GAIN_EDITING + for ( int16_t band_idx = 0; band_idx < MASA_FREQUENCY_BANDS; band_idx++ ) + { + for ( ch = 0; ch < 2; ch++ ) + { + hMasaIsmData->ismPreprocMatrix[ch][ch][band_idx] = 1.0f; + hMasaIsmData->ismPreprocMatrix[1 - ch][ch][band_idx] = 0.0f; + hMasaIsmData->eneMoveIIR[ch][band_idx] = 0.0f; + hMasaIsmData->enePreserveIIR[ch][band_idx] = 0.0f; + } + hMasaIsmData->eneOrigIIR[band_idx] = 0.0f; + hMasaIsmData->preprocEneTarget[band_idx] = 0.0f; + hMasaIsmData->preprocEneRealized[band_idx] = 0.0f; + } +#else for ( bin = 0; bin < CLDFB_NO_CHANNELS_MAX; bin++ ) { for ( ch = 0; ch < 2; ch++ ) @@ -81,15 +100,31 @@ ivas_error ivas_omasa_data_open( hMasaIsmData->preprocEneRealized[bin] = 0.0f; } +#endif + +#ifdef ENABLE_OBJECT_GAIN_EDITING + hMasaIsmData->objectsEdited = 0; +#else hMasaIsmData->objectsMoved = 0; +#endif hMasaIsmData->delayBuffer = NULL; for ( ch = 0; ch < MAX_NUM_OBJECTS; ch++ ) { +#ifdef ENABLE_OBJECT_EDITING + hMasaIsmData->ism_dir_is_edited[ch] = 0; +#else hMasaIsmData->ism_is_edited[ch] = 0; +#endif +#ifdef ENABLE_OBJECT_GAIN_EDITING + hMasaIsmData->ism_gain_is_edited[ch] = 0; +#endif hMasaIsmData->q_elevation_old[ch] = 0.0f; hMasaIsmData->q_azimuth_old[ch] = 0.0f; } +#ifdef ENABLE_OBJECT_GAIN_EDITING + hMasaIsmData->masa_gain_is_edited = 0; +#endif for ( obj_idx = 0; obj_idx < MAX_NUM_OBJECTS; obj_idx++ ) { @@ -199,7 +234,11 @@ ivas_error ivas_omasa_dec_config( #endif ) { +#ifdef OMASA_ISM_META_OPT + int16_t k, sce_id, nSCE_old, nchan_hp20_old, numCldfbAnalyses_old, numCldfbSyntheses_old; +#else int16_t k, sce_id, nSCE_old, nchan_hp20_old, numCldfbAnalyses_old, numCldfbSyntheses_old, n_MD; +#endif int32_t ivas_total_brate, ism_total_brate, cpe_brate; ISM_MODE ism_mode_old; IVAS_FORMAT ivas_format_orig; @@ -295,6 +334,16 @@ ivas_error ivas_omasa_dec_config( if ( ism_mode_old != st_ivas->ism_mode ) { /* ISM MD reconfig. */ +#ifdef OMASA_ISM_META_OPT + /* TODO: is it necessary to close the old structs if the number is constant? alternatively only check if allocation is needed */ + ivas_ism_metadata_close( st_ivas->hIsmMetaData, 0 ); + if ( ( error = ivas_ism_metadata_dec_create( st_ivas, st_ivas->nchan_ism, NULL ) ) != IVAS_ERR_OK ) + { + return error; + } + + ivas_ism_metadata_close( st_ivas->hIsmMetaData, st_ivas->nchan_ism ); /* close extra structs */ +#else n_MD = 0; if ( st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_EXTERNAL ) @@ -338,6 +387,7 @@ ivas_error ivas_omasa_dec_config( ivas_ism_metadata_close( st_ivas->hIsmMetaData, n_MD ); } +#endif st_ivas->hCPE[0]->element_brate = ivas_total_brate - ism_total_brate; @@ -706,17 +756,47 @@ void ivas_omasa_dirac_rend_jbm( if ( !st_ivas->hDecoderConfig->Opt_tsm ) { *nSamplesRendered = min( nSamplesAsked, st_ivas->hTcBuffer->n_samples_available ); - if ( st_ivas->ism_mode == ISM_MASA_MODE_MASA_ONE_OBJ || st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ ) { mvr2r( &output_f[CPE_CHANNELS][st_ivas->hTcBuffer->n_samples_rendered], data_separated_objects[0], *nSamplesRendered ); + +#ifdef ENABLE_OBJECT_GAIN_EDITING + if ( st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ ) + { + /* Gain separated object, if edited */ + for ( n = 0; n < st_ivas->nchan_ism; n++ ) + { + if ( st_ivas->hMasaIsmData->ism_gain_is_edited[n] && st_ivas->hMasaIsmData->idx_separated_ism == n ) + { + v_multc( data_separated_objects[0], st_ivas->hMasaIsmData->gain_ism_edited[n], data_separated_objects[0], *nSamplesRendered ); + } + } + } +#endif } else { for ( n = 0; n < st_ivas->nchan_ism; n++ ) { mvr2r( &output_f[n + CPE_CHANNELS][st_ivas->hTcBuffer->n_samples_rendered], data_separated_objects[n], *nSamplesRendered ); +#ifdef ENABLE_OBJECT_GAIN_EDITING + /* Gain discrete objects, if edited */ + if ( st_ivas->hMasaIsmData->ism_gain_is_edited[n] ) + { + v_multc( data_separated_objects[n], st_ivas->hMasaIsmData->gain_ism_edited[n], data_separated_objects[n], *nSamplesRendered ); + } +#endif } +#ifdef ENABLE_OBJECT_GAIN_EDITING + /* Gain MASA part, if edited */ + if ( st_ivas->hMasaIsmData->masa_gain_is_edited ) + { + for ( int16_t ch = 0; ch < 2; ch++ ) + { + v_multc( output_f[ch], st_ivas->hMasaIsmData->gain_masa_edited, output_f[ch], *nSamplesRendered ); + } + } +#endif } } diff --git a/lib_dec/ivas_stat_dec.h b/lib_dec/ivas_stat_dec.h index 4deb6748e2..f37e20ac16 100644 --- a/lib_dec/ivas_stat_dec.h +++ b/lib_dec/ivas_stat_dec.h @@ -896,7 +896,18 @@ typedef struct ivas_masa_ism_data_structure int16_t azimuth_ism_edited[MAX_NUM_OBJECTS]; int16_t elevation_ism_edited[MAX_NUM_OBJECTS]; + +#ifdef ENABLE_OBJECT_EDITING + uint8_t ism_dir_is_edited[MAX_NUM_OBJECTS]; +#else uint8_t ism_is_edited[MAX_NUM_OBJECTS]; +#endif +#ifdef ENABLE_OBJECT_GAIN_EDITING + float gain_ism_edited[MAX_NUM_OBJECTS]; + uint8_t ism_gain_is_edited[MAX_NUM_OBJECTS]; + float gain_masa_edited; + uint8_t masa_gain_is_edited; +#endif int16_t idx_separated_ism; int16_t azimuth_separated_ism[MAX_PARAM_SPATIAL_SUBFRAMES + DELAY_MASA_PARAM_DEC_SFR]; @@ -905,12 +916,22 @@ typedef struct ivas_masa_ism_data_structure float q_azimuth_old[MAX_NUM_OBJECTS]; float q_elevation_old[MAX_NUM_OBJECTS]; +#ifdef ENABLE_OBJECT_GAIN_EDITING + uint8_t objectsEdited; + float preprocEneTarget[MASA_FREQUENCY_BANDS]; + float preprocEneRealized[MASA_FREQUENCY_BANDS]; + float ismPreprocMatrix[2][2][MASA_FREQUENCY_BANDS]; + float eneMoveIIR[2][MASA_FREQUENCY_BANDS]; + float enePreserveIIR[2][MASA_FREQUENCY_BANDS]; + float eneOrigIIR[MASA_FREQUENCY_BANDS]; +#else float ismPreprocMatrix[2][2][CLDFB_NO_CHANNELS_MAX]; uint8_t objectsMoved; float eneMoveIIR[2][CLDFB_NO_CHANNELS_MAX]; float enePreserveIIR[2][CLDFB_NO_CHANNELS_MAX]; float preprocEneTarget[CLDFB_NO_CHANNELS_MAX]; float preprocEneRealized[CLDFB_NO_CHANNELS_MAX]; +#endif float **delayBuffer; int16_t delayBuffer_size; diff --git a/lib_dec/lib_dec.c b/lib_dec/lib_dec.c index 12bb331929..04e04868b2 100644 --- a/lib_dec/lib_dec.c +++ b/lib_dec/lib_dec.c @@ -982,18 +982,19 @@ ivas_error IVAS_DEC_GetSamples( nSamplesTcsScaled = hIvasDec->nSamplesFrame; } #ifdef ENABLE_OBJECT_EDITING - if ( ( hIvasDec->st_ivas->ivas_format == ISM_FORMAT || ( hIvasDec->st_ivas->ivas_format == SBA_ISM_FORMAT && hIvasDec->st_ivas->ism_mode == ISM_SBA_MODE_DISC ) || - ( hIvasDec->st_ivas->ivas_format == MASA_ISM_FORMAT && ( hIvasDec->st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ || hIvasDec->st_ivas->ism_mode == ISM_MASA_MODE_DISC ) ) || - ( hIvasDec->st_ivas->ivas_format == MASA_FORMAT && hIvasDec->st_ivas->nchan_ism > 0 ) ) && hIvasDec->hObjectEditCallback != NULL ) + ( hIvasDec->st_ivas->ivas_format == MASA_ISM_FORMAT && ( hIvasDec->st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ || hIvasDec->st_ivas->ism_mode == ISM_MASA_MODE_DISC ) ) ) && + hIvasDec->hObjectEditCallback != NULL && hIvasDec->st_ivas->nchan_ism > 0 ) { uint16_t ism_idx; /* copy object metadata into a simplified structure for providing outside */ IVAS_ISM_METADATA IsmMetadata[IVAS_MAX_NUM_OBJECTS]; - IVAS_ISM_METADATA* hIsmMetadata[IVAS_MAX_NUM_OBJECTS]; + IVAS_ISM_METADATA *hIsmMetadata[IVAS_MAX_NUM_OBJECTS]; bool mod_flag[IVAS_MAX_NUM_OBJECTS]; + float bed_gain; + bed_gain = 1.0f; for ( ism_idx = 0; ism_idx < IVAS_MAX_NUM_OBJECTS; ism_idx++ ) { @@ -1001,7 +1002,7 @@ ivas_error IVAS_DEC_GetSamples( } for ( ism_idx = 0; ism_idx < hIvasDec->st_ivas->nchan_ism; ism_idx++ ) { - hIsmMetadata[ism_idx] = &(IsmMetadata[ism_idx]); + hIsmMetadata[ism_idx] = &( IsmMetadata[ism_idx] ); if ( ( error = IVAS_DEC_GetObjectMetadata( hIvasDec, hIsmMetadata[ism_idx], 0, ism_idx ) ) != IVAS_ERR_OK ) { return error; @@ -1009,14 +1010,14 @@ ivas_error IVAS_DEC_GetSamples( } /* execute callback modifying the metadata */ - hIvasDec->hObjectEditCallback(hIsmMetadata, hIvasDec->st_ivas->nchan_ism, mod_flag ); + hIvasDec->hObjectEditCallback( hIsmMetadata, hIvasDec->st_ivas->nchan_ism, mod_flag, &bed_gain ); /* copy edited data from callback */ for ( ism_idx = 0; ism_idx < hIvasDec->st_ivas->nchan_ism; ism_idx++ ) { if ( mod_flag[ism_idx] ) { - IVAS_ISM_METADATA* srcIsmMeta = hIsmMetadata[ism_idx]; + IVAS_ISM_METADATA *srcIsmMeta = hIsmMetadata[ism_idx]; ISM_METADATA_HANDLE destIsmMeta = hIvasDec->st_ivas->hIsmMetaData[ism_idx]; destIsmMeta->azimuth = srcIsmMeta->azimuth; @@ -1045,8 +1046,17 @@ ivas_error IVAS_DEC_GetSamples( { hIvasDec->st_ivas->hMasaIsmData->elevation_ism_edited[ism_idx] = (int16_t) ( srcIsmMeta->elevation - 0.5f ); } - hIvasDec->st_ivas->hMasaIsmData->ism_is_edited[ism_idx] = 1u; - hIvasDec->st_ivas->hMasaIsmData->objectsMoved = 1u; + hIvasDec->st_ivas->hMasaIsmData->ism_dir_is_edited[ism_idx] = 1u; + + if ( fabsf( srcIsmMeta->gainFactor - 1.0f ) > OMASA_GAIN_EDIT_THR ) + { + hIvasDec->st_ivas->hMasaIsmData->ism_gain_is_edited[ism_idx] = 1u; + hIvasDec->st_ivas->hMasaIsmData->gain_ism_edited[ism_idx] = srcIsmMeta->gainFactor; + } + else + { + hIvasDec->st_ivas->hMasaIsmData->ism_gain_is_edited[ism_idx] = 0u; + } } } else @@ -1054,11 +1064,16 @@ ivas_error IVAS_DEC_GetSamples( if ( hIvasDec->st_ivas->hMasaIsmData != NULL ) { /* reset modification flag */ - hIvasDec->st_ivas->hMasaIsmData->ism_is_edited[ism_idx] = 0u; + hIvasDec->st_ivas->hMasaIsmData->ism_dir_is_edited[ism_idx] = 0u; } - } } + + if ( fabsf( bed_gain - 1.0f ) > OMASA_GAIN_EDIT_THR ) + { + hIvasDec->st_ivas->hMasaIsmData->gain_masa_edited = bed_gain; + hIvasDec->st_ivas->hMasaIsmData->masa_gain_is_edited = 1u; + } } #endif #ifdef DEBUG_MODE_JBM @@ -2349,11 +2364,11 @@ ivas_error IVAS_DEC_GetPcmFrameSize( #ifdef ENABLE_OBJECT_EDITING ivas_error IVAS_DEC_SetObjectEditCallback( - IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ - IvasObjEditCallbackFn callbackFn /* i : object editing callback function */ + IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ + IvasObjEditCallbackFn callbackFn /* i : object editing callback function */ ) { - if ( hIvasDec->st_ivas->ivas_format == ISM_FORMAT || hIvasDec->st_ivas->ivas_format == SBA_ISM_FORMAT || hIvasDec->st_ivas->ivas_format == MASA_ISM_FORMAT ) + if ( hIvasDec->st_ivas->ivas_format == ISM_FORMAT || hIvasDec->st_ivas->ivas_format == SBA_ISM_FORMAT || hIvasDec->st_ivas->ivas_format == MASA_ISM_FORMAT || hIvasDec->st_ivas->ivas_format == MASA_FORMAT ) { hIvasDec->hObjectEditCallback = callbackFn; return IVAS_ERR_OK; diff --git a/lib_dec/lib_dec.h b/lib_dec/lib_dec.h index 7b07da2d7d..c737c6856b 100644 --- a/lib_dec/lib_dec.h +++ b/lib_dec/lib_dec.h @@ -499,7 +499,7 @@ void IVAS_DEC_PrintConfig( ); #ifdef ENABLE_OBJECT_EDITING -typedef void (*IvasObjEditCallbackFn)(IVAS_ISM_METADATA*[IVAS_MAX_NUM_OBJECTS], uint16_t, bool* ); +typedef void (*IvasObjEditCallbackFn)(IVAS_ISM_METADATA*[IVAS_MAX_NUM_OBJECTS], uint16_t, bool*, float* ); ivas_error IVAS_DEC_SetObjectEditCallback( IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ IvasObjEditCallbackFn callbackFn /* i : object editing callback function */ diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index d3d50a6b71..47abd5a345 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -751,10 +751,17 @@ static void ivas_dirac_dec_binaural_internal( ivas_sba_prototype_renderer( st_ivas, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, subframe ); } +#ifdef ENABLE_OBJECT_GAIN_EDITING + if ( st_ivas->ivas_format == MASA_ISM_FORMAT && ( st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ || ( st_ivas->ism_mode == ISM_MASA_MODE_DISC && ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM || st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC ) ) ) ) + { + ivas_omasa_preProcessStereoTransportsForEditedObjects( st_ivas, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, nBins, subframe ); + } +#else if ( st_ivas->ivas_format == MASA_ISM_FORMAT && nchan_transport == 2 && st_ivas->ism_mode != ISM_MASA_MODE_DISC && st_ivas->ism_mode != ISM_MASA_MODE_MASA_ONE_OBJ ) { ivas_omasa_preProcessStereoTransportsForMovedObjects( st_ivas, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, nBins, subframe ); } +#endif if ( hCombinedOrientationData ) { @@ -1689,11 +1696,19 @@ static void ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matric uint16_t ismDirIndex; ismDirIndex = dirIndex - hSpatParamRendCom->numParametricDirections; assert( hMasaIsmData != NULL && "hMasaIsmData should not be NULL if we use it" ); +#ifdef ENABLE_OBJECT_EDITING + if ( hMasaIsmData->ism_dir_is_edited[ismDirIndex] ) + { + aziDeg = hMasaIsmData->azimuth_ism_edited[ismDirIndex]; + eleDeg = hMasaIsmData->elevation_ism_edited[ismDirIndex]; + } +#else if ( hMasaIsmData->ism_is_edited[ismDirIndex] ) { aziDeg = hMasaIsmData->azimuth_ism_edited[ismDirIndex]; eleDeg = hMasaIsmData->elevation_ism_edited[ismDirIndex]; } +#endif else { aziDeg = hMasaIsmData->azimuth_ism[ismDirIndex][dirac_read_idx]; @@ -2155,7 +2170,7 @@ static void ivas_dirac_dec_binaural_determine_processing_matrices( if ( ism_mode == ISM_MASA_MODE_DISC ) { #ifdef ENABLE_OBJECT_EDITING - if ( hMasaIsmData->ism_is_edited[chB] ) + if ( hMasaIsmData->ism_dir_is_edited[chB] ) { aziDeg = hMasaIsmData->azimuth_ism_edited[chB]; eleDeg = hMasaIsmData->elevation_ism_edited[chB]; @@ -2173,7 +2188,7 @@ static void ivas_dirac_dec_binaural_determine_processing_matrices( else { #ifdef ENABLE_OBJECT_EDITING - if ( ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ && hMasaIsmData->ism_is_edited[hMasaIsmData->idx_separated_ism] ) + if ( ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ && hMasaIsmData->ism_dir_is_edited[hMasaIsmData->idx_separated_ism] ) { aziDeg = hMasaIsmData->azimuth_ism_edited[hMasaIsmData->idx_separated_ism]; eleDeg = hMasaIsmData->elevation_ism_edited[hMasaIsmData->idx_separated_ism]; @@ -2981,6 +2996,53 @@ static void getDirectPartGains( if ( renderStereoOutputInsteadOfBinaural ) /* In stereo (i.e. non-binaural) rendering mode */ { +#ifdef STEREO_GAIN_CACHE + if ( aziDeg == gainCache->azi && eleDeg == gainCache->ele ) + { + *lRealp = gainCache->shVec[0]; /* Reused memory */ + *rRealp = gainCache->shVec[1]; /* Reused memory */ + } + else + { + /* Convert azi and ele to an azi value of the cone of confusion */ + aziRad = (float) aziDeg * PI_OVER_180; + eleRad = (float) eleDeg * PI_OVER_180; + y = ( sinf( aziRad ) * cosf( eleRad ) ); + mappedX = sqrtf( max( 0.0f, 1.0f - ( y * y ) ) ); + aziRadMapped = atan2f( y, mappedX ); + + /* Determine the real valued amplitude panning gains */ + *lImagp = 0.0f; + *rImagp = 0.0f; + if ( aziRadMapped >= LsAngleRad ) + { /* Left side */ + *lRealp = 1.0f; + *rRealp = 0.0f; + } + else if ( aziRadMapped <= -LsAngleRad ) + { /* Right side */ + *lRealp = 0.0f; + *rRealp = 1.0f; + } + else /* Tangent panning law */ + { + A = tanf( aziRadMapped ) / tanf( LsAngleRad ); + A2 = ( A - 1.0f ) / max( 0.001f, A + 1.0f ); + A3 = 1.0f / ( A2 * A2 + 1.0f ); + *lRealp = sqrtf( A3 ); + *rRealp = sqrtf( 1.0f - A3 ); + } + /* Scaling to have the same expected gain as for the HRTF rendering */ + *lRealp *= SQRT2; + *rRealp *= SQRT2; + + /* Store to gain cache */ + gainCache->azi = aziDeg; + gainCache->ele = eleDeg; + gainCache->shVec[0] = *lRealp; /* Reuse memory */ + gainCache->shVec[1] = *rRealp; /* Reuse memory */ + } +#else /* Convert azi and ele to an azi value of the cone of confusion */ aziRad = (float) aziDeg * PI_OVER_180; eleRad = (float) eleDeg * PI_OVER_180; @@ -3013,6 +3075,7 @@ static void getDirectPartGains( /* Scaling to have the same expected gain as for the HRTF rendering */ *lRealp *= SQRT2; *rRealp *= SQRT2; +#endif } else /* In regular binaural rendering mode */ { @@ -3165,7 +3228,492 @@ float configure_reqularization_factor( return reqularizationFactor; } +#ifdef ENABLE_OBJECT_GAIN_EDITING +/*-------------------------------------------------------------------* + * ivas_omasa_preProcessStereoTransportsForEditedObjects() + * + * + *-------------------------------------------------------------------*/ + +void ivas_omasa_preProcessStereoTransportsForEditedObjects( + Decoder_Struct *st_ivas, + float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + const int16_t nBins, + const int16_t subframe ) +{ + int16_t bin, ch, inCh, outCh, ismDirIndex, slot, band_idx, bin_lo, bin_hi, max_band; + SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom; + MASA_ISM_DATA_HANDLE hMasaIsmData; + uint8_t enableCentering; + int16_t dirac_read_idx; + int16_t nSlots; + float panGainsOut[4][2]; + float panGainsIn[4][2]; + float panEnesOut[4][2]; + float panEnesIn[4][2]; + uint8_t ismGainEdited[4]; + uint8_t ismDirEdited[4]; + uint8_t masaGainEdited; + + hSpatParamRendCom = st_ivas->hSpatParamRendCom; + hMasaIsmData = st_ivas->hMasaIsmData; + + if ( st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_FOA || st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_HOA2 || st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_HOA3 ) + { + enableCentering = 0; + } + else + { + enableCentering = 1; + } + + for ( ismDirIndex = 0; ismDirIndex < hSpatParamRendCom->numIsmDirections; ismDirIndex++ ) + { + ismGainEdited[ismDirIndex] = hMasaIsmData->ism_gain_is_edited[ismDirIndex]; + ismDirEdited[ismDirIndex] = hMasaIsmData->ism_dir_is_edited[ismDirIndex]; + } + masaGainEdited = hMasaIsmData->masa_gain_is_edited; + + + /* Bypass processing until first object is moved or gained */ + if ( hMasaIsmData->objectsEdited == 0 ) + { + for ( ismDirIndex = 0; ismDirIndex < hSpatParamRendCom->numIsmDirections; ismDirIndex++ ) + { + + if ( ismDirEdited[ismDirIndex] ) + { + hMasaIsmData->objectsEdited = 1; + } + + if ( ismGainEdited[ismDirIndex] ) + { + hMasaIsmData->objectsEdited = 1; + } + } + + if ( masaGainEdited ) + { + hMasaIsmData->objectsEdited = 1; + } + + if ( hMasaIsmData->objectsEdited == 0 ) + { + /* No objects have moved so far */ + return; + } + } + + /* OMASA gaining for discrete OMASA mode with stereo_param/bin_room_param renderer */ + if ( ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM || st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC ) && st_ivas->ism_mode == ISM_MASA_MODE_DISC ) + { + /* ISM gaining */ + for ( ch = 0; ch < st_ivas->nchan_ism; ch++ ) + { + if ( st_ivas->hMasaIsmData->ism_gain_is_edited[ch] ) + { + for ( slot = 0; slot < hSpatParamRendCom->subframe_nbslots[subframe]; slot++ ) + { + float gain; + gain = st_ivas->hMasaIsmData->gain_ism_edited[ch]; + v_multc( inRe[ch + 2][slot], gain, inRe[ch + 2][slot], nBins ); + v_multc( inIm[ch + 2][slot], gain, inIm[ch + 2][slot], nBins ); + } + } + } + /* MASA gaining */ + for ( ch = 0; ch < 2; ch++ ) + { + if ( masaGainEdited ) + { + for ( slot = 0; slot < hSpatParamRendCom->subframe_nbslots[subframe]; slot++ ) + { + float gain_masa; + gain_masa = st_ivas->hMasaIsmData->gain_masa_edited; + v_multc( inRe[ch][slot], gain_masa, inRe[ch][slot], nBins ); + v_multc( inIm[ch][slot], gain_masa, inIm[ch][slot], nBins ); + } + } + } + } + else /* Other processing modes */ + { + float subframeEne; + float subframeEneCh[2]; + float normEnes[2]; + float ratioAccOrig; + float ratioAccNew; + float ratio; + float ismEneThis; + float ismTargetEneThis; + float ismTargetEneThisCh[2]; + float totalTargetEneCh[2]; + float totalTargetEne; + float masaTargetEneThisCh[2]; + float ismPreprocMtxNew[2][2]; + float ismPreprocMtxIncrement[2][2]; + float eneMove[2]; + float enePreserve[2]; + float ismRatioAcc; + float remainderNormEne; + float centeringFactor; + float eneMoveThis; + float enePreserveThis; + float normVal; + float eqVal; + float outSlotRe[2][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float outSlotIm[2][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float newRatios[6]; + float gainIsmThis; + float gainMasaPow2; + float nSlotDiv; + float tempDivisor; + int16_t numIsm; + float masaEneThisCh[2]; + float ratioAccNewDivisor; + + + gainMasaPow2 = 1.0f; + if ( masaGainEdited ) + { + gainMasaPow2 = hMasaIsmData->gain_masa_edited; + gainMasaPow2 *= gainMasaPow2; + } + + nSlots = hSpatParamRendCom->subframe_nbslots[subframe]; + dirac_read_idx = hSpatParamRendCom->render_to_md_map[subframe]; + numIsm = hSpatParamRendCom->numIsmDirections; + nSlotDiv = 1.0f / ( (float) nSlots ); + + /* Determine panning gains and energies for each object */ + for ( ismDirIndex = 0; ismDirIndex < numIsm; ismDirIndex++ ) + { + /* Get input and output panning gains */ + ivas_get_stereo_panning_gains( hMasaIsmData->azimuth_ism[ismDirIndex][dirac_read_idx], + hMasaIsmData->elevation_ism[ismDirIndex][dirac_read_idx], + panGainsIn[ismDirIndex] ); + + if ( ismDirEdited[ismDirIndex] ) + { + ivas_get_stereo_panning_gains( hMasaIsmData->azimuth_ism_edited[ismDirIndex], + hMasaIsmData->elevation_ism_edited[ismDirIndex], + panGainsOut[ismDirIndex] ); + } + else + { + /* When not edited, input and output pan gains are the same */ + for ( ch = 0; ch < 2; ch++ ) + { + panGainsOut[ismDirIndex][ch] = panGainsIn[ismDirIndex][ch]; + } + } + /* Determine pan enes */ + for ( ch = 0; ch < 2; ch++ ) + { + panEnesOut[ismDirIndex][ch] = panGainsOut[ismDirIndex][ch] * panGainsOut[ismDirIndex][ch]; + panEnesIn[ismDirIndex][ch] = panGainsIn[ismDirIndex][ch] * panGainsIn[ismDirIndex][ch]; + } + } + + /* Determine the highest band */ + max_band = 0; + while ( max_band < MASA_FREQUENCY_BANDS && MASA_band_grouping_24[max_band] < nBins ) + { + max_band++; + } + + /* Init out array */ + for ( int k = 0; k < nSlots; k++ ) + { + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + set_zero( outSlotRe[ch][k], CLDFB_NO_CHANNELS_MAX ); + set_zero( outSlotIm[ch][k], CLDFB_NO_CHANNELS_MAX ); + } + } + + /* Perform the processing in frequency bands */ + for ( band_idx = 0; band_idx < max_band; band_idx++ ) + { + ratioAccOrig = 0.0f; + ratioAccNew = 0.0f; + ismRatioAcc = 0.0f; + + set_zero( subframeEneCh, 2 ); + set_zero( ismPreprocMtxNew[0], 2 ); + set_zero( ismPreprocMtxNew[1], 2 ); + set_zero( eneMove, 2 ); + set_zero( enePreserve, 2 ); + bin_lo = MASA_band_grouping_24[band_idx]; + bin_hi = min( MASA_band_grouping_24[band_idx + 1], (int16_t) nBins ); + + /* Determine transport normalized energies and subframe energy */ + for ( slot = 0; slot < nSlots; slot++ ) + { + for ( ch = 0; ch < 2; ch++ ) + { + for ( bin = bin_lo; bin < bin_hi; bin++ ) + { + subframeEneCh[ch] += inRe[ch][slot][bin] * inRe[ch][slot][bin]; + subframeEneCh[ch] += inIm[ch][slot][bin] * inIm[ch][slot][bin]; + } + } + } + subframeEne = subframeEneCh[0] + subframeEneCh[1]; + totalTargetEneCh[0] = subframeEneCh[0]; + totalTargetEneCh[1] = subframeEneCh[1]; + masaEneThisCh[0] = subframeEneCh[0]; + masaEneThisCh[1] = subframeEneCh[1]; + + /* Gain editing */ + /* For each object, estimate new target energy per channel based on the applied gain */ + for ( ismDirIndex = 0; ismDirIndex < numIsm; ismDirIndex++ ) + { + ratio = hMasaIsmData->energy_ratio_ism[ismDirIndex][dirac_read_idx][bin_lo]; + ratioAccOrig += ratio; + /* Calculate MASA energy as a residual of original channel energies subtracted with ISM energies */ + for ( ch = 0; ch < 2; ch++ ) + { + masaEneThisCh[ch] -= panEnesIn[ismDirIndex][ch] * ratio * subframeEne; + } + + /* Calculate target energy, gained ratio for accumulation, and transports gains, if ism gain is edited */ + if ( ismGainEdited[ismDirIndex] ) + { + gainIsmThis = hMasaIsmData->gain_ism_edited[ismDirIndex]; + /* ISM original energy */ + ismEneThis = ratio * subframeEne; + + /* ISM target energy */ + ismTargetEneThis = ( gainIsmThis * gainIsmThis ) * ismEneThis; + + ratio *= gainIsmThis * gainIsmThis; + + /* Determine panning energies and channel target energies */ + for ( ch = 0; ch < 2; ch++ ) + { + ismTargetEneThisCh[ch] = panEnesIn[ismDirIndex][ch] * ismTargetEneThis; /* Ism target energy per channel */ + totalTargetEneCh[ch] -= panEnesIn[ismDirIndex][ch] * ismEneThis; /* Reduce original ism energy */ + totalTargetEneCh[ch] += ismTargetEneThisCh[ch]; /* Add ism target energy per channel */ + } + + /* If separated ism edited, apply gain directly to the separated ism */ + if ( ismDirIndex == hMasaIsmData->idx_separated_ism ) + { + /* Separated object gaining is done elsewhere with DIRAC renderer */ + if ( st_ivas->renderer_type != RENDERER_DIRAC ) + { + /* Gain transport channel of separated ism */ + for ( slot = 0; slot < nSlots; slot++ ) + { + for ( bin = bin_lo; bin < bin_hi; bin++ ) + { + inRe[2][slot][bin] = gainIsmThis * inRe[2][slot][bin]; + inIm[2][slot][bin] = gainIsmThis * inIm[2][slot][bin]; + } + } + } + } + } + ratioAccNew += ratio; + } + + /* MASA original ratios */ + ratio = hSpatParamRendCom->energy_ratio1[dirac_read_idx][bin_lo] + hSpatParamRendCom->energy_ratio2[dirac_read_idx][bin_lo]; + ratioAccOrig += ratio; + + /* Estimate MASA original energies and determine adjusted target energies and gained ratio for accumulation, if a gain is applied to MASA */ + if ( masaGainEdited ) + { + /* Gained MASA ratio */ + ratio *= gainMasaPow2; + + /* Calculate MASA target energies and add to total target energy estimation */ + for ( ch = 0; ch < 2; ch++ ) + { + masaEneThisCh[ch] = fmaxf( masaEneThisCh[ch], 0.0f ); /* MASA original energy per channel */ + masaTargetEneThisCh[ch] = gainMasaPow2 * masaEneThisCh[ch]; /* MASA target energy per channel */ + totalTargetEneCh[ch] -= masaEneThisCh[ch]; /* Reduce original energy per channel */ + totalTargetEneCh[ch] += masaTargetEneThisCh[ch]; /* Add target energy per channel */ + } + } + + for ( ch = 0; ch < 2; ch++ ) + { + totalTargetEneCh[ch] = max( totalTargetEneCh[ch], 0.0f ); + } + + ratioAccNew += ratio; + + /* Diffuse ratio accumulation based on gaining */ + if ( masaGainEdited ) + { + ratioAccNew += gainMasaPow2 * ( 1 - ratioAccOrig ); + } + else + { + ratioAccNew += ( 1.0f - ratioAccOrig ); + } + + ratioAccNewDivisor = 1.0f / fmaxf( 1e-12f, ratioAccNew ); /* New target total energy ratio divider */ + + /* Determine and process object energy ratios based on gaining */ + for ( ismDirIndex = 0; ismDirIndex < numIsm; ismDirIndex++ ) + { + newRatios[ismDirIndex + 2] = hMasaIsmData->energy_ratio_ism[ismDirIndex][dirac_read_idx][bin_lo]; /* Determine original object energy ratio */ + if ( ismGainEdited[ismDirIndex] ) + { + gainIsmThis = hMasaIsmData->gain_ism_edited[ismDirIndex]; + newRatios[ismDirIndex + 2] *= ( gainIsmThis * gainIsmThis ); /* Gain original object energy ratio, if edited */ + } + newRatios[ismDirIndex + 2] *= ratioAccNewDivisor; /* Divide with new target total ratio */ + } + + /* Determine and process MASA energy ratios based on gaining */ + newRatios[0] = hSpatParamRendCom->energy_ratio1[dirac_read_idx][bin_lo]; + newRatios[1] = hSpatParamRendCom->energy_ratio2[dirac_read_idx][bin_lo]; + if ( masaGainEdited ) + { + newRatios[0] *= gainMasaPow2; + newRatios[1] *= gainMasaPow2; + } + newRatios[0] *= ratioAccNewDivisor; + newRatios[1] *= ratioAccNewDivisor; + + /* Set adjusted energy ratios */ + for ( bin = bin_lo; bin < bin_hi; bin++ ) + { + for ( ismDirIndex = 0; ismDirIndex < numIsm; ismDirIndex++ ) + { + hMasaIsmData->energy_ratio_ism[ismDirIndex][dirac_read_idx][bin] = newRatios[ismDirIndex + 2]; + } + hSpatParamRendCom->energy_ratio1[dirac_read_idx][bin] = newRatios[0]; + hSpatParamRendCom->energy_ratio2[dirac_read_idx][bin] = newRatios[1]; + } + + /* Direction editing */ + /* Determine new energetic values after gaining */ + totalTargetEne = totalTargetEneCh[0] + totalTargetEneCh[1]; + tempDivisor = 1.0f / fmaxf( 1e-12f, totalTargetEne ); + normEnes[0] = totalTargetEneCh[0] * tempDivisor; + normEnes[1] = totalTargetEneCh[1] * tempDivisor; + + /* For each ismDir, determine moved and preserve energy ratio per channel */ + for ( ismDirIndex = 0; ismDirIndex < numIsm; ismDirIndex++ ) + { + ratio = hMasaIsmData->energy_ratio_ism[ismDirIndex][dirac_read_idx][bin_lo]; + + ismRatioAcc += ratio; + + if ( enableCentering ) + { + centeringFactor = fmaxf( 0.0f, 2.0f * fabsf( panEnesIn[ismDirIndex][0] - panEnesOut[ismDirIndex][0] ) - 1.0f ); + for ( ch = 0; ch < 2; ch++ ) + { + panEnesOut[ismDirIndex][ch] *= ( 1.0f - centeringFactor ); + panEnesOut[ismDirIndex][ch] += 0.5f * centeringFactor; + } + } + + for ( ch = 0; ch < 2; ch++ ) + { + eneMoveThis = fmaxf( 0.0f, panEnesIn[ismDirIndex][ch] - panEnesOut[ismDirIndex][ch] ); + enePreserveThis = panEnesIn[ismDirIndex][ch] - eneMoveThis; + + eneMove[ch] += ratio * eneMoveThis; + enePreserve[ch] += ratio * enePreserveThis; + + /* Subtract object parts from normEnes */ + normEnes[ch] -= panEnesIn[ismDirIndex][ch] * ratio; + } + } + + /* Any remaining (non-object) energy is set to be preserved at both channels */ + remainderNormEne = fmaxf( 0.0f, ( 1.0f - ismRatioAcc ) - normEnes[0] - normEnes[1] ); + + /* Normalize */ + for ( ch = 0; ch < 2; ch++ ) + { + enePreserve[ch] += fmaxf( 0.0f, normEnes[ch] + remainderNormEne / 2.0f ); + normVal = 1.0f / fmaxf( EPSILON, eneMove[ch] + enePreserve[ch] ); + normVal *= fminf( 10.0f, totalTargetEneCh[ch] / fmaxf( EPSILON, subframeEneCh[ch] ) ); + eneMove[ch] *= normVal; + enePreserve[ch] *= normVal; + } + + /* Temporally average target energy */ + hMasaIsmData->eneOrigIIR[band_idx] *= STEREO_PREPROCESS_IIR_FACTOR; + hMasaIsmData->eneOrigIIR[band_idx] += totalTargetEne; + + /* Temporally average energy moving and preserving, and generate the transport signal preprocessing matrix for + * gaining objects and moving objects between left and right */ + for ( ch = 0; ch < 2; ch++ ) + { + hMasaIsmData->eneMoveIIR[ch][band_idx] *= STEREO_PREPROCESS_IIR_FACTOR; + hMasaIsmData->eneMoveIIR[ch][band_idx] += eneMove[ch] * totalTargetEne; + hMasaIsmData->enePreserveIIR[ch][band_idx] *= STEREO_PREPROCESS_IIR_FACTOR; + hMasaIsmData->enePreserveIIR[ch][band_idx] += enePreserve[ch] * totalTargetEne; + normVal = 1.0f / fmaxf( EPSILON, hMasaIsmData->eneOrigIIR[band_idx] ); + ismPreprocMtxNew[ch][ch] = fminf( 4.0f, sqrtf( hMasaIsmData->enePreserveIIR[ch][band_idx] * normVal ) ); + ismPreprocMtxNew[1 - ch][ch] = fminf( 4.0f, sqrtf( hMasaIsmData->eneMoveIIR[ch][band_idx] * normVal ) ); + } + + /* Get increment value for temporal interpolation */ + for ( inCh = 0; inCh < 2; inCh++ ) + { + for ( outCh = 0; outCh < 2; outCh++ ) + { + ismPreprocMtxIncrement[outCh][inCh] = ( ismPreprocMtxNew[outCh][inCh] - hMasaIsmData->ismPreprocMatrix[outCh][inCh][band_idx] ) * nSlotDiv; + } + } + + /* Mix signals */ + hMasaIsmData->preprocEneTarget[band_idx] *= STEREO_PREPROCESS_IIR_FACTOR; + hMasaIsmData->preprocEneRealized[band_idx] *= STEREO_PREPROCESS_IIR_FACTOR; + hMasaIsmData->preprocEneTarget[band_idx] += totalTargetEne; + + for ( outCh = 0; outCh < 2; outCh++ ) + { + for ( slot = 0; slot < nSlots; slot++ ) + { + for ( inCh = 0; inCh < 2; inCh++ ) + { + hMasaIsmData->ismPreprocMatrix[outCh][inCh][band_idx] += ismPreprocMtxIncrement[outCh][inCh]; + for ( bin = bin_lo; bin < bin_hi; bin++ ) + { + outSlotRe[outCh][slot][bin] += inRe[inCh][slot][bin] * hMasaIsmData->ismPreprocMatrix[outCh][inCh][band_idx]; + outSlotIm[outCh][slot][bin] += inIm[inCh][slot][bin] * hMasaIsmData->ismPreprocMatrix[outCh][inCh][band_idx]; + } + } + + for ( bin = bin_lo; bin < bin_hi; bin++ ) + { + hMasaIsmData->preprocEneRealized[band_idx] += ( outSlotRe[outCh][slot][bin] * outSlotRe[outCh][slot][bin] ) + ( outSlotIm[outCh][slot][bin] * outSlotIm[outCh][slot][bin] ); + } + } + } + + eqVal = fminf( 4.0f, sqrtf( hMasaIsmData->preprocEneTarget[band_idx] / fmaxf( 1e-12f, hMasaIsmData->preprocEneRealized[band_idx] ) ) ); + + for ( ch = 0; ch < 2; ch++ ) + { + for ( slot = 0; slot < nSlots; slot++ ) + { + for ( bin = bin_lo; bin < bin_hi; bin++ ) + { + inRe[ch][slot][bin] = outSlotRe[ch][slot][bin] * eqVal; + inIm[ch][slot][bin] = outSlotIm[ch][slot][bin] * eqVal; + } + } + } + } + } + + return; +} +#else /*-------------------------------------------------------------------* * ivas_omasa_preProcessStereoTransportsForMovedObjects() * @@ -3390,6 +3938,8 @@ void ivas_omasa_preProcessStereoTransportsForMovedObjects( return; } +#endif + static void ivas_masa_ext_rend_parambin_internal( MASA_EXT_REND_HANDLE hMasaExtRend, diff --git a/lib_rend/ivas_dirac_output_synthesis_dec.c b/lib_rend/ivas_dirac_output_synthesis_dec.c index 54ea302165..38afce083c 100644 --- a/lib_rend/ivas_dirac_output_synthesis_dec.c +++ b/lib_rend/ivas_dirac_output_synthesis_dec.c @@ -1700,10 +1700,17 @@ void ivas_dirac_dec_compute_directional_responses( for ( dir = 0; dir < hSpatParamRendCom->numIsmDirections; dir++ ) { +#ifdef ENABLE_OBJECT_EDITING + if ( hMasaIsm->ism_dir_is_edited[dir] ) + { + ivas_dirac_dec_get_response( hMasaIsm->azimuth_ism_edited[dir], hMasaIsm->elevation_ism_edited[dir], direct_response_temp, hDirACRend->hOutSetup.ambisonics_order ); + } +#else if ( hMasaIsm->ism_is_edited[dir] ) { ivas_dirac_dec_get_response( hMasaIsm->azimuth_ism_edited[dir], hMasaIsm->elevation_ism_edited[dir], direct_response_temp, hDirACRend->hOutSetup.ambisonics_order ); } +#endif else { ivas_dirac_dec_get_response( hMasaIsm->azimuth_ism[dir][md_idx], hMasaIsm->elevation_ism[dir][md_idx], direct_response_temp, hDirACRend->hOutSetup.ambisonics_order ); @@ -1810,10 +1817,17 @@ void ivas_dirac_dec_compute_directional_responses( for ( dir = 0; dir < hSpatParamRendCom->numIsmDirections; dir++ ) { +#ifdef ENABLE_OBJECT_EDITING + if ( hMasaIsm->ism_dir_is_edited[dir] ) + { + vbap_determine_gains( hVBAPdata, direct_response_temp, hMasaIsm->azimuth_ism_edited[dir], hMasaIsm->elevation_ism_edited[dir], 1 ); + } +#else if ( hMasaIsm->ism_is_edited[dir] ) { vbap_determine_gains( hVBAPdata, direct_response_temp, hMasaIsm->azimuth_ism_edited[dir], hMasaIsm->elevation_ism_edited[dir], 1 ); } +#endif else { vbap_determine_gains( hVBAPdata, direct_response_temp, hMasaIsm->azimuth_ism[dir][md_idx], hMasaIsm->elevation_ism[dir][md_idx], 1 ); -- GitLab From e7076c86453c26488ffbe867df107a530445a4fe Mon Sep 17 00:00:00 2001 From: Jouni Paulus Date: Thu, 18 Apr 2024 09:03:58 +0200 Subject: [PATCH 3/7] object edit part missing from split rendering path --- lib_rend/ivas_dirac_dec_binaural_functions.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index 47abd5a345..dbae49698b 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -1254,11 +1254,19 @@ static void ivas_dirac_dec_binaural_formulate_target_covariance_matrices( uint16_t ismDirIndex; ismDirIndex = dirIndex - hSpatParamRendCom->numParametricDirections; assert( hMasaIsmData != NULL && "hMasaIsmData should not be NULL if we use it" ); +#ifdef ENABLE_OBJECT_EDITING + if ( hMasaIsmData->ism_dir_is_edited[ismDirIndex] ) + { + aziDeg = hMasaIsmData->azimuth_ism_edited[ismDirIndex]; + eleDeg = hMasaIsmData->elevation_ism_edited[ismDirIndex]; + } +#else if ( hMasaIsmData->ism_is_edited[ismDirIndex] ) { aziDeg = hMasaIsmData->azimuth_ism_edited[ismDirIndex]; eleDeg = hMasaIsmData->elevation_ism_edited[ismDirIndex]; } +#endif else { aziDeg = hMasaIsmData->azimuth_ism[ismDirIndex][dirac_read_idx]; -- GitLab From 95a64278826f6a82dc63e40e8d46c25f9050378a Mon Sep 17 00:00:00 2001 From: Jouni Paulus Date: Thu, 18 Apr 2024 11:07:07 +0200 Subject: [PATCH 4/7] exclude object edit callback call from WMC_TOOL to allow instrumented builds --- lib_dec/lib_dec.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib_dec/lib_dec.c b/lib_dec/lib_dec.c index 04e04868b2..870e0ed062 100644 --- a/lib_dec/lib_dec.c +++ b/lib_dec/lib_dec.c @@ -1009,8 +1009,10 @@ ivas_error IVAS_DEC_GetSamples( } } +#define WMC_TOOL_SKIP /* execute callback modifying the metadata */ hIvasDec->hObjectEditCallback( hIsmMetadata, hIvasDec->st_ivas->nchan_ism, mod_flag, &bed_gain ); +#undef WMC_TOOL_SKIP /* copy edited data from callback */ for ( ism_idx = 0; ism_idx < hIvasDec->st_ivas->nchan_ism; ism_idx++ ) -- GitLab From 1782344cbd048d82c6dc2c0495ebf04c54a31b25 Mon Sep 17 00:00:00 2001 From: lintervo Date: Thu, 18 Apr 2024 13:11:58 +0000 Subject: [PATCH 5/7] fix MSAN-error --- lib_rend/ivas_dirac_dec_binaural_functions.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index dbae49698b..9b89cadc04 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -3005,6 +3005,8 @@ static void getDirectPartGains( if ( renderStereoOutputInsteadOfBinaural ) /* In stereo (i.e. non-binaural) rendering mode */ { #ifdef STEREO_GAIN_CACHE + *lImagp = 0.0f; + *rImagp = 0.0f; if ( aziDeg == gainCache->azi && eleDeg == gainCache->ele ) { *lRealp = gainCache->shVec[0]; /* Reused memory */ @@ -3020,8 +3022,6 @@ static void getDirectPartGains( aziRadMapped = atan2f( y, mappedX ); /* Determine the real valued amplitude panning gains */ - *lImagp = 0.0f; - *rImagp = 0.0f; if ( aziRadMapped >= LsAngleRad ) { /* Left side */ *lRealp = 1.0f; -- GitLab From 94935228f1717a40bd5a35c56980d826dd59c5d4 Mon Sep 17 00:00:00 2001 From: "Arvi Lintervo (Nokia)" Date: Mon, 22 Apr 2024 10:22:23 +0300 Subject: [PATCH 6/7] ism metadata decoder reconfig changes for enabling object editing with bitrate switching --- lib_dec/ivas_masa_dec.c | 4 ++++ lib_dec/ivas_omasa_dec.c | 18 ++++++++++++------ lib_dec/lib_dec.c | 10 ++++++---- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/lib_dec/ivas_masa_dec.c b/lib_dec/ivas_masa_dec.c index 3d05c01ad4..e9eb04b1e1 100644 --- a/lib_dec/ivas_masa_dec.c +++ b/lib_dec/ivas_masa_dec.c @@ -596,7 +596,11 @@ ivas_error ivas_masa_dec_open( ism_total_brate = 0; /* ISM metadata */ +#ifdef OMASA_ISM_META_OPT + if ( st_ivas->ivas_format == MASA_FORMAT && st_ivas->hIsmMetaData[0] != NULL ) +#else if ( st_ivas->ivas_format == MASA_FORMAT && st_ivas->hIsmMetaData[0] != NULL && st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_EXTERNAL ) +#endif { /* these are not needed -> clean. EXT metafile writer in OMASA needs only the number of ISMs and writes default null-data */ ivas_ism_metadata_close( st_ivas->hIsmMetaData, 0 ); diff --git a/lib_dec/ivas_omasa_dec.c b/lib_dec/ivas_omasa_dec.c index d105956212..4313258aaa 100644 --- a/lib_dec/ivas_omasa_dec.c +++ b/lib_dec/ivas_omasa_dec.c @@ -335,14 +335,20 @@ ivas_error ivas_omasa_dec_config( { /* ISM MD reconfig. */ #ifdef OMASA_ISM_META_OPT - /* TODO: is it necessary to close the old structs if the number is constant? alternatively only check if allocation is needed */ - ivas_ism_metadata_close( st_ivas->hIsmMetaData, 0 ); - if ( ( error = ivas_ism_metadata_dec_create( st_ivas, st_ivas->nchan_ism, NULL ) ) != IVAS_ERR_OK ) + if ( st_ivas->hIsmMetaData[0] == NULL ) { - return error; + if ( ( error = ivas_ism_metadata_dec_create( st_ivas, st_ivas->nchan_ism, NULL ) ) != IVAS_ERR_OK ) + { + return error; + } + } + else + { + for ( k = 0; k < st_ivas->nchan_ism; k++ ) + { + ivas_ism_reset_metadata_handle_dec( st_ivas->hIsmMetaData[k] ); + } } - - ivas_ism_metadata_close( st_ivas->hIsmMetaData, st_ivas->nchan_ism ); /* close extra structs */ #else n_MD = 0; diff --git a/lib_dec/lib_dec.c b/lib_dec/lib_dec.c index 870e0ed062..a4ebe7f2e1 100644 --- a/lib_dec/lib_dec.c +++ b/lib_dec/lib_dec.c @@ -1070,11 +1070,13 @@ ivas_error IVAS_DEC_GetSamples( } } } - - if ( fabsf( bed_gain - 1.0f ) > OMASA_GAIN_EDIT_THR ) + if ( hIvasDec->st_ivas->ivas_format == MASA_ISM_FORMAT ) { - hIvasDec->st_ivas->hMasaIsmData->gain_masa_edited = bed_gain; - hIvasDec->st_ivas->hMasaIsmData->masa_gain_is_edited = 1u; + if ( fabsf( bed_gain - 1.0f ) > OMASA_GAIN_EDIT_THR ) + { + hIvasDec->st_ivas->hMasaIsmData->gain_masa_edited = bed_gain; + hIvasDec->st_ivas->hMasaIsmData->masa_gain_is_edited = 1u; + } } } #endif -- GitLab From bb4991bcff36c8d564078061f57ba1781533631b Mon Sep 17 00:00:00 2001 From: "Arvi Lintervo (Nokia)" Date: Mon, 22 Apr 2024 10:50:07 +0300 Subject: [PATCH 7/7] clang format --- lib_dec/ivas_init_dec.c | 1 - 1 file changed, 1 deletion(-) diff --git a/lib_dec/ivas_init_dec.c b/lib_dec/ivas_init_dec.c index 85120c2ed7..7e72dd7b83 100644 --- a/lib_dec/ivas_init_dec.c +++ b/lib_dec/ivas_init_dec.c @@ -1700,7 +1700,6 @@ ivas_error ivas_init_decoder( } #endif #endif - } else if ( st_ivas->ism_mode == ISM_MASA_MODE_DISC ) { -- GitLab