diff --git a/CMakeLists.txt b/CMakeLists.txt index a048f400a67e211190f73c7a5e41f2014ab20224..317274ddce0b47fa97ce1455dc0715671aa6dd95 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -145,7 +145,7 @@ target_link_libraries(lib_enc lib_com lib_debug) file(GLOB libRendSrcs "lib_rend/*.c") file(GLOB libRendHeaders "lib_rend/*.h") add_library(lib_rend ${libRendSrcs} ${libRendHeaders}) -target_link_libraries(lib_rend lib_com lib_debug) +target_link_libraries(lib_rend lib_dec lib_com lib_debug) # Todo refactor: This dependency on lib_dec should be removed. file(GLOB libDecSrcs "lib_dec/*.c") file(GLOB libDecHeaders "lib_dec/*.h") diff --git a/Makefile b/Makefile index 9514e9a582437a1e60735a29b3a0d5c637d560c0..141a607b075279da49e4530e701a7f5453303bb0 100644 --- a/Makefile +++ b/Makefile @@ -184,8 +184,8 @@ $(CLI_APIENC): $(OBJS_CLI_APIENC) $(LIB_LIBENC) $(LIB_LIBCOM) $(LIB_LIBUTIL) $(L $(CLI_APIDEC): $(OBJS_CLI_APIDEC) $(LIB_LIBDEC) $(LIB_LIBCOM) $(LIB_LIBUTIL) $(LIB_LIBDEBUG) $(QUIET_LINK)$(CC) $(LDFLAGS) $(OBJS_CLI_APIDEC) -L. -livasdec -livascom -livasutil -livasdebug $(LDLIBS) -o $(CLI_APIDEC) -$(CLI_APIREND): $(OBJS_CLI_APPREND) $(LIB_LIBREND) $(LIB_LIBCOM) $(LIB_LIBUTIL) $(LIB_LIBDEBUG) - $(QUIET_LINK)$(CC) $(LDFLAGS) $(OBJS_CLI_APPREND) -L. -livasrend -livasutil -livasdebug -livascom $(LDLIBS) -o $(CLI_APIREND) +$(CLI_APIREND): $(OBJS_CLI_APPREND) $(LIB_LIBREND) $(LIB_LIBCOM) $(LIB_LIBUTIL) $(LIB_LIBDEBUG) $(LIB_LIBDEC) + $(QUIET_LINK)$(CC) $(LDFLAGS) $(OBJS_CLI_APPREND) -L. -livasrend -livasdec -livasutil -livasdebug -livascom $(LDLIBS) -o $(CLI_APIREND) $(CLI_UTESTS_CREND): $(OBJS_CLI_UTESTS_CREND) $(LIB_LIBDEC) $(LIB_LIBCOM) $(LIB_LIBUTIL) $(LIB_LIBDEBUG) $(QUIET_LINK)$(CC) $(LDFLAGS) $(OBJS_CLI_UTESTS_CREND) -L. -livasdec -livascom -livasutil -livasdebug $(LDLIBS) -o $(UTESTS_CREND_DIR)/$(CLI_UTESTS_CREND) diff --git a/apps/renderer.c b/apps/renderer.c index 8097424cec34869269043c7360a1a457fed15216..0d46dd05f496dcdef21d8ed1fa819110b19d5314 100644 --- a/apps/renderer.c +++ b/apps/renderer.c @@ -424,7 +424,12 @@ static int16_t getTotalNumInChannels( IVAS_REND_HANDLE hIvasRend, IVAS_REND_InputId mcIds[RENDERER_MAX_MC_INPUTS], IVAS_REND_InputId ismIds[RENDERER_MAX_ISM_INPUTS], - IVAS_REND_InputId sbaIds[RENDERER_MAX_SBA_INPUTS] ) + IVAS_REND_InputId sbaIds[RENDERER_MAX_SBA_INPUTS] +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + , + IVAS_REND_InputId masaIds[RENDERER_MAX_MASA_INPUTS] +#endif +) { int16_t totalNumInChannels = 0; int16_t i, numInputChannels; @@ -479,6 +484,24 @@ static int16_t getTotalNumInChannels( totalNumInChannels += numInputChannels; } +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + for ( int32_t i = 0; i < RENDERER_MAX_MASA_INPUTS; ++i ) + { + if ( masaIds[i] == 0 ) + { + /* Skip inactive inputs */ + continue; + } + + if ( ( error = IVAS_REND_GetInputNumChannels( hIvasRend, masaIds[i], &numInputChannels ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + totalNumInChannels += numInputChannels; + } +#endif + return totalNumInChannels; } @@ -651,6 +674,9 @@ int main( IVAS_REND_InputId mcIds[RENDERER_MAX_MC_INPUTS] = { 0 }; IVAS_REND_InputId ismIds[RENDERER_MAX_ISM_INPUTS] = { 0 }; IVAS_REND_InputId sbaIds[RENDERER_MAX_SBA_INPUTS] = { 0 }; +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + IVAS_REND_InputId masaIds[RENDERER_MAX_MASA_INPUTS] = { 0 }; +#endif if ( ( error = IVAS_REND_Open( &hIvasRend, args.sampleRate, args.outConfig.audioConfig ) ) != IVAS_ERR_OK ) { @@ -759,7 +785,29 @@ int main( } } +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + for ( i = 0; i < args.inConfig.numMasaBuses; ++i ) + { + if ( ( error = IVAS_REND_AddInput( hIvasRend, args.inConfig.masaBuses[i].audioConfig, &masaIds[i] ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + + if ( ( error = IVAS_REND_SetInputGain( hIvasRend, masaIds[i], args.inputGainGlobal * dBToLin( args.inConfig.masaBuses[i].gain_dB ) ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + } +#endif + +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + const int16_t totalNumInChannels = getTotalNumInChannels( hIvasRend, mcIds, ismIds, sbaIds, masaIds ); +#else const int16_t totalNumInChannels = getTotalNumInChannels( hIvasRend, mcIds, ismIds, sbaIds ); +#endif + if ( AudioFileReader_getNumChannels( audioReader ) != 0 /* If input file is raw PCM, audio reader has no info about number of channels */ && totalNumInChannels != AudioFileReader_getNumChannels( audioReader ) ) { @@ -855,6 +903,7 @@ int main( } #endif +#ifndef NOKIA_MASA_EXTERNAL_RENDERER for ( i = 0; i < RENDERER_MAX_MASA_INPUTS; ++i ) { if ( masaReaders[i] != NULL ) @@ -865,6 +914,7 @@ int main( (void) hMasaMetadata; } } +#endif ObjectPositionBuffer mtdBuffer; IsmPositionProvider_getNextFrame( positionProvider, &mtdBuffer ); @@ -930,6 +980,37 @@ int main( } } +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + for ( i = 0; i < args.inConfig.numMasaBuses; ++i ) + { + int16_t numChannels; + if ( ( error = IVAS_REND_GetInputNumChannels( hIvasRend, masaIds[i], &numChannels ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + IVAS_REND_ReadOnlyAudioBuffer tmpBuffer = getReadOnlySubBuffer( inBuffer, args.inConfig.masaBuses[i].inputChannelIndex, numChannels ); + + if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, masaIds[i], tmpBuffer ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + + if ( masaReaders[i] != NULL ) + { + /* This will update data in hMasaMetadata[i] */ + MasaFileReader_readNextFrame( masaReaders[i] ); + + if ( ( error = IVAS_REND_FeedInputMasaMetadata( hIvasRend, masaIds[i], hMasaMetadata[i] ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + } + } +#endif + IVAS_REND_GetSamples( hIvasRend, outBuffer ); int16_t num_out_channels; @@ -1307,6 +1388,8 @@ static IVAS_REND_AudioConfig parseAudioConfig( switch ( charBuf[4] ) { case '1': + fprintf( stderr, "1TC MASA support is not functional and is pending on DirAC renderer refactoring.\n" ); + exit( EXIT_FAILURE ); return IVAS_REND_AUDIO_CONFIG_MASA1; case '2': return IVAS_REND_AUDIO_CONFIG_MASA2; diff --git a/lib_com/ivas_error.h b/lib_com/ivas_error.h index f0435aaac2a26a4e5f241890c7ee53f68cc42c71..fdc7a1502df931a6e4e08151525d141169808540 100644 --- a/lib_com/ivas_error.h +++ b/lib_com/ivas_error.h @@ -63,6 +63,9 @@ typedef enum IVAS_ERR_TOO_MANY_INPUTS, #else IVAS_ERR_TOO_MANY_OBJECT_INPUTS, +#endif +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + IVAS_ERR_MISSING_METADATA, #endif IVAS_ERR_INDEX_OUT_OF_BOUNDS, IVAS_ERR_RECONFIGURE_NOT_SUPPORTED, diff --git a/lib_com/options.h b/lib_com/options.h index a7ddce2788d0232c2d8e4d5ec4f2b55dae8e2133..4b71f20a255112e0799fb84159bc5bf6933f81d9 100644 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -146,6 +146,7 @@ #define FIX_I1_113 /* under review : MCT bit distribution optimization for SBA high bitrates*/ #define PRINT_SBA_ORDER /* Issue 179: print-out also the SBA order of IVAS SBA format to stdout */ #define EXT_RENDERER /* FhG: external renderer library and standalone application */ +#define NOKIA_MASA_EXTERNAL_RENDERER /* Nokia: MASA support for external renderer */ #define FIX_EFAP_MATH /* fix for EFAP: remove angle quantization and a bug in polygon lookup causing incorrect gains. minor tweak for ALLRAD. non-BE for modes using EFAP */ #define FIX_124_DONT_ALLOC_PLCINFO_IN_IVAS /* Issue 124: do not allocate unused plc struct in IVAS modes which is only used in EVS mono */ #define FIX_MCT_PLC_RECOVERY /* Issue 184: scale the old synthesis part correctly in the first good frame after lost frames in MCT modes - to be activated after previous switch is merged */ @@ -154,6 +155,7 @@ #define REMOVE_SID_HARM_LEFTOVERS /* Issue 192: remove leftovers from the SID bitrate harmonization */ + /* ################## End DEVELOPMENT switches ######################### */ /* clang-format on */ #endif diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index ef38d00c19e3513c9d6a00e66740cfcc1647e68e..caff4d52c02aa1924e311d883455297a98702921 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -140,6 +140,20 @@ typedef struct rotation_gains rot_gains_prev; } input_sba; +#ifdef NOKIA_MASA_EXTERNAL_RENDERER +/* Due to API of some rendering methods, the renderer has to use the decoder struct. + Only struct members relevant for rendering will be initialized, therefore typedef as "dummy" decoder struct */ +typedef Decoder_Struct DecoderDummy; + +typedef struct +{ + input_base base; + DecoderDummy *decDummy; + MASA_METADATA_FRAME masaMetadata; + bool metadataHasBeenFed; +} input_masa; +#endif + struct IVAS_REND { int32_t sampleRateOut; @@ -152,6 +166,9 @@ struct IVAS_REND input_ism inputsIsm[RENDERER_MAX_ISM_INPUTS]; input_mc inputsMc[RENDERER_MAX_MC_INPUTS]; input_sba inputsSba[RENDERER_MAX_SBA_INPUTS]; +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + input_masa inputsMasa[RENDERER_MAX_MASA_INPUTS]; +#endif /* TODO @Philips - inputConfig should not be stored here, but read from e.g. input_mc->input_base.inConfig, please remove this */ IVAS_REND_AudioConfig inputConfig; @@ -383,11 +400,13 @@ ivas_error getAudioConfigNumChannels( { case IVAS_REND_AUDIO_CONFIG_MONO: case IVAS_REND_AUDIO_CONFIG_OBJECT: + case IVAS_REND_AUDIO_CONFIG_MASA1: *numChannels = 1; break; case IVAS_REND_AUDIO_CONFIG_STEREO: case IVAS_REND_AUDIO_CONFIG_BINAURAL: case IVAS_REND_AUDIO_CONFIG_BINAURAL_ROOM: + case IVAS_REND_AUDIO_CONFIG_MASA2: *numChannels = 2; break; case IVAS_REND_AUDIO_CONFIG_FOA: @@ -1849,6 +1868,427 @@ static void clearInputSba( return; } +#ifdef NOKIA_MASA_EXTERNAL_RENDERER +static ivas_error initMasaDummyDecForMcOut( input_masa *inputMasa, IVAS_REND_AudioConfig outConfig ) +{ + ivas_error error; + int16_t numCldfbAnalyses; + int16_t numCldfbSyntheses; + int16_t i; + AUDIO_CONFIG output_config; + DecoderDummy *decDummy; + decDummy = inputMasa->decDummy; + + output_config = rendAudioConfigToIvasAudioConfig( outConfig ); + decDummy->hDecoderConfig->output_config = output_config; + + decDummy->hDecoderConfig->ivas_total_brate = IVAS_512k; /* Todo Nokia: This is preventing initialization of 2TC as 1TC, should be fixed properly in ivas_dirac_dec_config() */ + decDummy->sba_mode = SBA_MODE_NONE; /* Todo Nokia: This is done to prevent ivas_dirac_dec_config() to not use uninitialized value. It could be considered if this should not be even accessed when not in SBA. */ + decDummy->mc_mode = MC_MODE_NONE; /* Todo Nokia: This should be also refactored in such way that it is not checked if not in MC mode */ + + ivas_output_init( &( decDummy->hOutSetup ), output_config ); + ivas_output_init( &( decDummy->hIntSetup ), output_config ); + + decDummy->renderer_type = RENDERER_DIRAC; + if ( output_config == AUDIO_CONFIG_STEREO ) + { + decDummy->renderer_type = RENDERER_STEREO_PARAMETRIC; + } + + decDummy->ivas_format = MASA_FORMAT; + decDummy->transport_config = AUDIO_CONFIG_INVALID; + + /* Todo refactor: Access to qmetadata is not required by the algorithm. */ + if ( ( error = ivas_qmetadata_open( &( decDummy->hQMetaData ) ) ) != IVAS_ERR_OK ) + { + return error; + } + decDummy->hQMetaData->coherence_flag = 1; + + if ( ( error = ivas_dirac_dec_open( decDummy ) ) != IVAS_ERR_OK ) + { + return error; + } + + if ( decDummy->renderer_type == RENDERER_STEREO_PARAMETRIC ) + { + if ( ( error = ivas_dirac_dec_init_binaural_data( decDummy ) ) != IVAS_ERR_OK ) + { + return error; + } + decDummy->hDiracDecBin->useSubframeMode = 0; /* Todo Nokia: This will disappear in later work but needs to be this now. */ + } + + numCldfbAnalyses = decDummy->nchan_transport; + numCldfbSyntheses = decDummy->hDecoderConfig->nchan_out; + + for ( i = 0; i < numCldfbAnalyses; i++ ) + { + if ( ( error = openCldfb( &( decDummy->cldfbAnaDec[i] ), CLDFB_ANALYSIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + { + return error; + } + } + for ( ; i < MAX_INTERN_CHANNELS; i++ ) + { + decDummy->cldfbAnaDec[i] = NULL; + } + + for ( i = 0; i < numCldfbSyntheses; i++ ) + { + if ( ( error = openCldfb( &( decDummy->cldfbSynDec[i] ), CLDFB_SYNTHESIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + { + return error; + } + } + for ( ; i < MAX_OUTPUT_CHANNELS; i++ ) + { + decDummy->cldfbSynDec[i] = NULL; + } + + return IVAS_ERR_OK; +} + +static ivas_error initMasaDummyDecForSbaOut( input_masa *inputMasa, IVAS_REND_AudioConfig outConfig ) +{ + ivas_error error; + int16_t numCldfbAnalyses; + int16_t numCldfbSyntheses; + int16_t i; + AUDIO_CONFIG output_config; + DecoderDummy *decDummy; + + decDummy = inputMasa->decDummy; + + output_config = rendAudioConfigToIvasAudioConfig( outConfig ); + decDummy->hDecoderConfig->output_config = output_config; + + decDummy->hDecoderConfig->ivas_total_brate = IVAS_512k; /* Todo Nokia: This is preventing initialization of 2TC as 1TC, should be fixed properly in ivas_dirac_dec_config() */ + decDummy->sba_mode = SBA_MODE_NONE; /* Todo Nokia: This is done to prevent ivas_dirac_dec_config() to not use uninitialized value. It could be considered if this should not be even accessed when not in SBA. */ + decDummy->mc_mode = MC_MODE_NONE; /* Todo Nokia: This should be also refactored in such way that it is not checked if not in MC mode */ + + ivas_output_init( &( decDummy->hOutSetup ), output_config ); + ivas_output_init( &( decDummy->hIntSetup ), output_config ); + decDummy->renderer_type = RENDERER_DIRAC; + decDummy->ivas_format = MASA_FORMAT; + decDummy->transport_config = AUDIO_CONFIG_INVALID; + + /* Todo refactor: Access to qmetadata is not required by the algorithm. */ + if ( ( error = ivas_qmetadata_open( &( decDummy->hQMetaData ) ) ) != IVAS_ERR_OK ) + { + return error; + } + decDummy->hQMetaData->coherence_flag = 1; + + if ( ( error = ivas_dirac_dec_open( decDummy ) ) != IVAS_ERR_OK ) + { + return error; + } + + numCldfbAnalyses = decDummy->nchan_transport; + numCldfbSyntheses = decDummy->hDecoderConfig->nchan_out; + + for ( i = 0; i < numCldfbAnalyses; i++ ) + { + if ( ( error = openCldfb( &( decDummy->cldfbAnaDec[i] ), CLDFB_ANALYSIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + { + return error; + } + } + for ( ; i < MAX_INTERN_CHANNELS; i++ ) + { + decDummy->cldfbAnaDec[i] = NULL; + } + + for ( i = 0; i < numCldfbSyntheses; i++ ) + { + if ( ( error = openCldfb( &( decDummy->cldfbSynDec[i] ), CLDFB_SYNTHESIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + { + return error; + } + } + for ( ; i < MAX_OUTPUT_CHANNELS; i++ ) + { + decDummy->cldfbSynDec[i] = NULL; + } + + return IVAS_ERR_OK; +} + +static ivas_error initMasaDummyDecForBinauralOut( input_masa *inputMasa, IVAS_REND_AudioConfig outConfig ) +{ + ivas_error error; + + int16_t i; + AUDIO_CONFIG output_config; + DecoderDummy *decDummy; + + decDummy = inputMasa->decDummy; + + output_config = rendAudioConfigToIvasAudioConfig( outConfig ); + decDummy->hDecoderConfig->output_config = output_config; + + output_config = decDummy->hDecoderConfig->output_config; + + decDummy->hDecoderConfig->ivas_total_brate = IVAS_512k; /* Todo Nokia: This is preventing initialization of 2TC as 1TC, should be fixed properly in ivas_dirac_dec_config() */ + decDummy->sba_mode = SBA_MODE_NONE; /* Todo Nokia: This is done to prevent ivas_dirac_dec_config() to not use uninitialized value. It could be considered if this should not be even accessed when not in SBA. */ + decDummy->mc_mode = MC_MODE_NONE; /* Todo Nokia: This should be also refactored in such way that it is not checked if not in MC mode */ + + ivas_output_init( &( decDummy->hOutSetup ), output_config ); + if ( output_config == AUDIO_CONFIG_BINAURAL ) + { + decDummy->renderer_type = RENDERER_BINAURAL_PARAMETRIC; + } + else + { + decDummy->renderer_type = RENDERER_BINAURAL_PARAMETRIC_ROOM; + } + decDummy->ivas_format = MASA_FORMAT; + decDummy->transport_config = AUDIO_CONFIG_INVALID; +#ifdef DEBUGGING + decDummy->hDecoderConfig->forceSubframeBinauralization = 0; +#endif + + if ( ( error = ivas_dirac_dec_open( decDummy ) ) != IVAS_ERR_OK ) + { + return error; + } + if ( ( error = ivas_dirac_dec_init_binaural_data( decDummy ) ) != IVAS_ERR_OK ) + { + return error; + } + + decDummy->hDiracDecBin->useSubframeMode = 0; /* Todo Nokia: This will disappear in later work but needs to be this now. */ + + for ( i = 0; i < BINAURAL_CHANNELS; i++ ) + { + if ( ( error = openCldfb( &( decDummy->cldfbAnaDec[i] ), CLDFB_ANALYSIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + { + return error; + } + if ( ( error = openCldfb( &( decDummy->cldfbSynDec[i] ), CLDFB_SYNTHESIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + { + return error; + } + } + for ( ; i < MAX_INTERN_CHANNELS; i++ ) + { + decDummy->cldfbAnaDec[i] = NULL; + decDummy->cldfbSynDec[i] = NULL; + } + + return IVAS_ERR_OK; +} + +static ivas_error updateMasaDummyDec( input_masa *inputMasa, IVAS_REND_AudioConfig outConfig ) +{ + ivas_error error; + + switch ( getAudioConfigType( outConfig ) ) + { + case IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED: + error = initMasaDummyDecForMcOut( inputMasa, outConfig ); + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: + error = initMasaDummyDecForSbaOut( inputMasa, outConfig ); + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL: + error = initMasaDummyDecForBinauralOut( inputMasa, outConfig ); + break; + default: + return IVAS_ERR_INVALID_OUTPUT_FORMAT; + } + /* Check error here to keep switch statement more compact */ + if ( error != IVAS_ERR_OK ) + { + return error; + } + + return IVAS_ERR_OK; +} + +static DecoderDummy *initDecoderDummy( int32_t sampleRate, int16_t numTransChannels, IVAS_REND_AudioConfig outConfig, const uint8_t enableRenderConfig ) +{ + ivas_error error; + int16_t i; + int16_t numOutChannels; + DecoderDummy *decDummy; + + if ( ( error = getAudioConfigNumChannels( outConfig, &numOutChannels ) ) != IVAS_ERR_OK ) + { + /* Checking error with assert is enough, this function is only temporary anyway */ + assert( error == IVAS_ERR_OK ); + } + + decDummy = count_malloc( sizeof( DecoderDummy ) ); + decDummy->hDecoderConfig = count_malloc( sizeof( DECODER_CONFIG ) ); + decDummy->hDecoderConfig->output_Fs = sampleRate; + decDummy->hDecoderConfig->nchan_out = (int16_t) numOutChannels; + decDummy->hDecoderConfig->Opt_Headrotation = 0; + + decDummy->hBinRenderer = NULL; + decDummy->hEFAPdata = NULL; + decDummy->hHrtf = NULL; + decDummy->hHrtfTD = NULL; + decDummy->hSpar = NULL; + decDummy->hoa_dec_mtx = NULL; + decDummy->hVBAPdata = NULL; + decDummy->hMasa = NULL; + decDummy->hDiracDecBin = NULL; + decDummy->hQMetaData = NULL; + decDummy->hDecoderConfig->output_config = rendAudioConfigToIvasAudioConfig( outConfig ); + decDummy->nchan_transport = numTransChannels; + + if ( outConfig == IVAS_REND_AUDIO_CONFIG_BINAURAL_ROOM || outConfig == IVAS_REND_AUDIO_CONFIG_BINAURAL_ROOM ) + { + decDummy->hHeadTrackData = count_malloc( sizeof( HEAD_TRACK_DATA ) ); + /* Initialise Rmat_prev to I, Rmat will be computed later */ + for ( i = 0; i < 3; i++ ) + { + set_zero( decDummy->hHeadTrackData->Rmat_prev[i], 3 ); + decDummy->hHeadTrackData->Rmat_prev[i][i] = 1.0f; + } + + decDummy->hHeadTrackData->num_quaternions = 0; + decDummy->hHeadTrackData->lrSwitchInterpVal = 0.0f; + decDummy->hHeadTrackData->lrSwitchedCurrent = 0; + decDummy->hHeadTrackData->lrSwitchedNext = 0; + } + else + { + decDummy->hHeadTrackData = NULL; + } + + if ( enableRenderConfig ) + { + ivas_render_config_open( &decDummy->hRenderConfig ); + decDummy->hRenderConfig->roomAcoustics.late_reverb_on = 0; + decDummy->hRenderConfig->roomAcoustics.use_brir = 0; + } + else + { + decDummy->hRenderConfig = NULL; + } + + decDummy->renderer_type = RENDERER_DISABLE; + + return decDummy; +} + +static ivas_error setRendInputActiveMasa( + void *input, + const IVAS_REND_AudioConfig inConfig, + const IVAS_REND_InputId id, + RENDER_CONFIG_DATA *hRendCfg ) /* Todo: This is not used at all within MASA. Support might be better to do after refactoring. */ +{ + ivas_error error; + rendering_context rendCtx; + IVAS_REND_AudioConfig outConfig; + input_masa *inputMasa; + int16_t numInChannels; + + inputMasa = (input_masa *) input; + rendCtx = inputMasa->base.ctx; + outConfig = *rendCtx.pOutConfig; + (void) hRendCfg; /* Suppress warning */ + + initRendInputBase( &inputMasa->base, inConfig, id, rendCtx ); + + if ( ( error = getAudioConfigNumChannels( inConfig, &numInChannels ) ) != IVAS_ERR_OK ) + { + return error; + } + inputMasa->decDummy = initDecoderDummy( *rendCtx.pOutSampleRate, (int16_t) numInChannels, outConfig, 0 ); + inputMasa->metadataHasBeenFed = false; + + if ( ( error = updateMasaDummyDec( inputMasa, outConfig ) ) != IVAS_ERR_OK ) + { + return error; + } + + return IVAS_ERR_OK; +} + +static void freeDecoderDummy( DecoderDummy **ppDecDummy ) +{ + int16_t i; + DecoderDummy *pDecDummy; + + if ( ppDecDummy == NULL || *ppDecDummy == NULL ) + { + return; + } + pDecDummy = *ppDecDummy; + + if ( pDecDummy->hDecoderConfig != NULL ) + { + count_free( pDecDummy->hDecoderConfig ); + } + if ( pDecDummy->hHeadTrackData != NULL ) + { + count_free( pDecDummy->hHeadTrackData ); + } + ivas_render_config_close( &pDecDummy->hRenderConfig ); + +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + /* CLDFB handles */ + for ( i = 0; i < MAX_INTERN_CHANNELS; i++ ) + { + if ( pDecDummy->cldfbAnaDec[i] != NULL ) + { + deleteCldfb( &( pDecDummy->cldfbAnaDec[i] ) ); + pDecDummy->cldfbAnaDec[i] = NULL; + } + } + + for ( i = 0; i < MAX_OUTPUT_CHANNELS; i++ ) + { + if ( pDecDummy->cldfbSynDec[i] != NULL ) + { + deleteCldfb( &( pDecDummy->cldfbSynDec[i] ) ); + pDecDummy->cldfbSynDec[i] = NULL; + } + } + + /* DirAC handle */ + if ( pDecDummy->hDirAC != NULL ) + { + ivas_dirac_dec_close( pDecDummy->hDirAC ); + pDecDummy->hDirAC = NULL; + } + + /* Qmetadata handle */ + ivas_qmetadata_close( &pDecDummy->hQMetaData ); + + /* VBAP handle */ + vbap_free_data( &( pDecDummy->hVBAPdata ) ); + + /* HOA decoder matrix */ + if ( pDecDummy->hoa_dec_mtx != NULL ) + { + count_free( pDecDummy->hoa_dec_mtx ); + pDecDummy->hoa_dec_mtx = NULL; + } + + /* Parametric binaural renderer handle */ + ivas_dirac_dec_close_binaural_data( &pDecDummy->hDiracDecBin ); +#endif + + count_free( pDecDummy ); + pDecDummy = NULL; +} + +static void clearInputMasa( input_masa *inputMasa ) +{ + rendering_context rendCtx; + + rendCtx = inputMasa->base.ctx; + + initRendInputBase( &inputMasa->base, IVAS_REND_AUDIO_CONFIG_UNKNOWN, 0, rendCtx ); + freeDecoderDummy( &inputMasa->decDummy ); +} +#endif + ivas_error IVAS_REND_Open( IVAS_REND_HANDLE *phIvasRend, const int32_t outputSampleRate, @@ -1925,6 +2365,17 @@ ivas_error IVAS_REND_Open( initRendInputBase( &hIvasRend->inputsSba[i].base, IVAS_REND_AUDIO_CONFIG_UNKNOWN, 0, getRendCtx( hIvasRend ) ); hIvasRend->inputsSba[i].crendWrapper.hCrend = NULL; } +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + for ( i = 0; i < RENDERER_MAX_MASA_INPUTS; ++i ) + { + initRendInputBase( &hIvasRend->inputsMasa[i].base, + IVAS_REND_AUDIO_CONFIG_UNKNOWN, + 0, + getRendCtx( hIvasRend ) ); + hIvasRend->inputsMasa[i].decDummy = NULL; + hIvasRend->inputsMasa[i].metadataHasBeenFed = false; + } +#endif return IVAS_ERR_OK; } @@ -2145,6 +2596,15 @@ static ivas_error getInputById( } pInputBase = &hIvasRend->inputsSba[inputIndex].base; break; +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: + if ( inputIndex > RENDERER_MAX_MASA_INPUTS ) + { + return IVAS_ERR_INVALID_INPUT_ID; + } + pInputBase = &hIvasRend->inputsMasa[inputIndex].base; + break; +#endif default: return IVAS_ERR_INVALID_INPUT_ID; } @@ -2203,6 +2663,15 @@ static ivas_error getConstInputById( } pInputBase = &hIvasRend->inputsSba[inputIndex].base; break; +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: + if ( inputIndex > RENDERER_MAX_MASA_INPUTS ) + { + return IVAS_ERR_INVALID_INPUT_ID; + } + pInputBase = &hIvasRend->inputsMasa[inputIndex].base; + break; +#endif default: return IVAS_ERR_INVALID_INPUT_ID; } @@ -2226,7 +2695,7 @@ static ivas_error findFreeInputSlot( int32_t *inputIndex ) { /* Using a void pointer and a separately provided size is a hack for this function - to be reusable for arrays of any input type (input_ism, input_mc, input_sba). + to be reusable for arrays of any input type (input_ism, input_mc, input_sba, input_masa). Assumptions: - input_base is always the first member in the input struct - provided size is correct @@ -2301,6 +2770,14 @@ ivas_error IVAS_REND_AddInput( inputStructSize = sizeof( *hIvasRend->inputsSba ); activateInput = setRendInputActiveSba; break; +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: + maxNumInputsOfType = RENDERER_MAX_MASA_INPUTS; + inputsArray = hIvasRend->inputsMasa; + inputStructSize = sizeof( *hIvasRend->inputsMasa ); + activateInput = setRendInputActiveMasa; + break; +#endif default: return IVAS_ERR_INVALID_INPUT_FORMAT; } @@ -2495,6 +2972,11 @@ ivas_error IVAS_REND_RemoveInput( case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: clearInputSba( (input_sba *) inputBase ); break; +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: + clearInputMasa( (input_masa *) inputBase ); + break; +#endif default: return IVAS_ERR_INVALID_INPUT_FORMAT; } @@ -2585,6 +3067,17 @@ ivas_error IVAS_REND_GetDelay( } } +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + for ( i = 0; i < RENDERER_MAX_MASA_INPUTS; i++ ) + { + if ( hIvasRend->inputsMasa[i].base.inConfig != IVAS_REND_AUDIO_CONFIG_UNKNOWN ) + { + latency_ns = NS2SA( hIvasRend->sampleRateOut, (int32_t) ( (float) IVAS_FB_DEC_DELAY_NS + 0.5f ) ); + *nSamples = max( *nSamples, NS2SA( *timeScale, latency_ns ) ); + } + } +#endif + return IVAS_ERR_OK; } @@ -2676,6 +3169,42 @@ ivas_error IVAS_REND_FeedInputObjectMetadata( return IVAS_ERR_OK; } +#ifdef NOKIA_MASA_EXTERNAL_RENDERER +ivas_error IVAS_REND_FeedInputMasaMetadata( + IVAS_REND_HANDLE hIvasRend, + const IVAS_REND_InputId inputId, + IVAS_MASA_METADATA_HANDLE masaMetadata ) +{ + ivas_error error; + input_base *inputBase; + input_masa *inputMasa; + + /*-----------------------------------------------------------------* + * Validate function arguments + *-----------------------------------------------------------------*/ + + if ( hIvasRend == NULL ) + { + return IVAS_ERR_UNEXPECTED_NULL_POINTER; + } + if ( ( error = getInputById( hIvasRend, inputId, (void **) &inputBase ) ) != IVAS_ERR_OK ) + { + return error; + } + if ( getAudioConfigType( inputBase->inConfig ) != IVAS_REND_AUDIO_CONFIG_TYPE_MASA ) + { + /* MASA metadata should only be fed for MASA inputs */ + return IVAS_ERR_METADATA_NOT_EXPECTED; + } + + inputMasa = (input_masa *) inputBase; + inputMasa->masaMetadata = *masaMetadata; + inputMasa->metadataHasBeenFed = true; + + return IVAS_ERR_OK; +} +#endif + ivas_error IVAS_REND_InitConfig( IVAS_REND_HANDLE st, bool rendererConfigEnabled ) { @@ -4011,6 +4540,178 @@ static ivas_error renderActiveInputsSba( return IVAS_ERR_OK; } +#ifdef NOKIA_MASA_EXTERNAL_RENDERER +static void copyMasaMetadataToDiracRenderer( MASA_METADATA_FRAME *meta, DIRAC_DEC_HANDLE hDirAC ) +{ + int16_t band, sf, bin; + + hDirAC->numSimultaneousDirections = meta->descriptive_meta.numberOfDirections + 1; + + for ( sf = 0; sf < MAX_PARAM_SPATIAL_SUBFRAMES; sf++ ) + { + for ( band = 0; band < MASA_MAXIMUM_CODING_SUBBANDS; band++ ) + { + for ( bin = MASA_band_grouping_24[band]; bin < MASA_band_grouping_24[band + 1]; bin++ ) + { + hDirAC->azimuth[sf][bin] = (int16_t) meta->directional_meta[0].azimuth[sf][band]; + hDirAC->elevation[sf][bin] = (int16_t) meta->directional_meta[0].elevation[sf][band]; + hDirAC->energy_ratio1[sf][bin] = meta->directional_meta[0].energy_ratio[sf][band]; + hDirAC->diffuseness_vector[sf][bin] = 1.0f - meta->directional_meta[0].energy_ratio[sf][band]; + hDirAC->spreadCoherence[sf][bin] = meta->directional_meta[0].spread_coherence[sf][band]; + hDirAC->surroundingCoherence[sf][bin] = meta->common_meta.surround_coherence[sf][band]; + + if ( hDirAC->numSimultaneousDirections == 2 ) + { + hDirAC->azimuth2[sf][bin] = (int16_t) meta->directional_meta[1].azimuth[sf][band]; + hDirAC->elevation2[sf][bin] = (int16_t) meta->directional_meta[1].elevation[sf][band]; + hDirAC->energy_ratio2[sf][bin] = meta->directional_meta[1].energy_ratio[sf][band]; + hDirAC->diffuseness_vector[sf][bin] -= meta->directional_meta[1].energy_ratio[sf][band]; + hDirAC->spreadCoherence2[sf][bin] = meta->directional_meta[1].spread_coherence[sf][band]; + } + } + } + } +} + +static ivas_error renderMasaToMc( input_masa *masaInput, IVAS_REND_AudioBuffer outAudio ) +{ + float tmpBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + + copyBufferTo2dArray( masaInput->base.inputBuffer, tmpBuffer ); + copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->decDummy->hDirAC ); + + /* TODO(sgi): Remove code duplication w.r.t. MASA rendering to other output configs */ + if ( masaInput->decDummy->renderer_type == RENDERER_STEREO_PARAMETRIC ) + { + ivas_dirac_dec_binaural( masaInput->decDummy, tmpBuffer, masaInput->base.inputBuffer.config.numChannels ); + } + else + { + ivas_dirac_dec( masaInput->decDummy, tmpBuffer, masaInput->base.inputBuffer.config.numChannels, NULL, NULL, -1 ); + } + + accumulate2dArrayToBuffer( tmpBuffer, &outAudio ); + + return IVAS_ERR_OK; +} + +static ivas_error renderMasaToSba( input_masa *masaInput, IVAS_REND_AudioBuffer outAudio ) +{ + float tmpBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + + copyBufferTo2dArray( masaInput->base.inputBuffer, tmpBuffer ); + copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->decDummy->hDirAC ); + + ivas_dirac_dec( masaInput->decDummy, tmpBuffer, masaInput->base.inputBuffer.config.numChannels, NULL, NULL, -1 ); + + accumulate2dArrayToBuffer( tmpBuffer, &outAudio ); + + return IVAS_ERR_OK; +} + +static ivas_error renderMasaToBinaural( input_masa *masaInput, IVAS_REND_AudioBuffer outAudio ) +{ + float tmpBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + + copyBufferTo2dArray( masaInput->base.inputBuffer, tmpBuffer ); + copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->decDummy->hDirAC ); + + ivas_dirac_dec_binaural( masaInput->decDummy, tmpBuffer, masaInput->base.inputBuffer.config.numChannels ); + + accumulate2dArrayToBuffer( tmpBuffer, &outAudio ); + + return IVAS_ERR_OK; +} + +static ivas_error renderInputMasa( + input_masa *masaInput, + IVAS_REND_AudioConfig outConfig, + IVAS_REND_AudioBuffer outAudio ) +{ + ivas_error error; + IVAS_REND_AudioBuffer inAudio; + + if ( !masaInput->metadataHasBeenFed ) + { + return IVAS_ERR_MISSING_METADATA; + } + + inAudio = masaInput->base.inputBuffer; + + if ( masaInput->base.numNewSamplesPerChannel != outAudio.config.numSamplesPerChannel ) + { + /* Mismatch between the number of input samples vs number of requested output samples - currently not allowed */ + return IVAS_ERR_INVALID_BUFFER_SIZE; + } + masaInput->base.numNewSamplesPerChannel = 0; + + /* Apply input gain to new audio */ + v_multc( inAudio.data, + masaInput->base.gain, + inAudio.data, + inAudio.config.numSamplesPerChannel * inAudio.config.numChannels ); + + switch ( getAudioConfigType( outConfig ) ) + { + case IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED: + error = renderMasaToMc( masaInput, outAudio ); + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: + error = renderMasaToSba( masaInput, outAudio ); + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL: + switch ( outConfig ) + { + case IVAS_REND_AUDIO_CONFIG_BINAURAL: + error = renderMasaToBinaural( masaInput, outAudio ); + break; + /* ToDo */ + // case IVAS_REND_AUDIO_CONFIG_BINAURAL_ROOM: + // error = renderMasaToBinauralRoom( masaInput, outConfig, outAudio ); + // break; + default: + return IVAS_ERR_INVALID_OUTPUT_FORMAT; + } + break; + default: + return IVAS_ERR_INVALID_OUTPUT_FORMAT; + } + /* Check error here to keep switch statement more compact */ + if ( error != IVAS_ERR_OK ) + { + return error; + } + + return IVAS_ERR_OK; +} + +static ivas_error renderActiveInputsMasa( + IVAS_REND_HANDLE hIvasRend, + IVAS_REND_AudioBuffer outAudio ) +{ + int32_t i; + input_masa *pCurrentInput; + ivas_error error; + + for ( i = 0, pCurrentInput = hIvasRend->inputsMasa; i < RENDERER_MAX_MASA_INPUTS; ++i, ++pCurrentInput ) + { + if ( pCurrentInput->base.inConfig == IVAS_REND_AUDIO_CONFIG_UNKNOWN ) + { + /* Skip inactive inputs */ + continue; + } + if ( ( error = renderInputMasa( pCurrentInput, + hIvasRend->outputConfig, + outAudio ) ) != IVAS_ERR_OK ) + { + return error; + } + } + + return IVAS_ERR_OK; +} +#endif + ivas_error IVAS_REND_GetSamples( IVAS_REND_HANDLE hIvasRend, IVAS_REND_AudioBuffer outAudio ) @@ -4064,6 +4765,12 @@ ivas_error IVAS_REND_GetSamples( { return error; } +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + if ( ( error = renderActiveInputsMasa( hIvasRend, outAudio ) ) != IVAS_ERR_OK ) + { + return error; + } +#endif #ifdef DEBUGGING hIvasRend->numClipping += @@ -4107,6 +4814,12 @@ void IVAS_REND_Close( { clearInputSba( &hIvasRend->inputsSba[i] ); } +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + for ( i = 0; i < RENDERER_MAX_MASA_INPUTS; ++i ) + { + clearInputMasa( &hIvasRend->inputsMasa[i] ); + } +#endif /* clear Config. Renderer */ ivas_render_config_close( &( hIvasRend->hRendererConfig ) ); diff --git a/lib_rend/lib_rend.h b/lib_rend/lib_rend.h index 99d4b7af25f6079d0b29367129ef6d5b99af4f06..158a734f4633a43ee65fb5a2a177a794684c3b7a 100644 --- a/lib_rend/lib_rend.h +++ b/lib_rend/lib_rend.h @@ -229,14 +229,22 @@ ivas_error IVAS_REND_FeedInputObjectMetadata( const IVAS_REND_AudioObjectPosition objectPosition /* i : object position struct */ ); +#ifdef NOKIA_MASA_EXTERNAL_RENDERER +ivas_error IVAS_REND_FeedInputMasaMetadata( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + const IVAS_REND_InputId inputId, /* i : ID of the input */ + IVAS_MASA_METADATA_HANDLE masaMetadata /* i : MASA metadata frame */ +); +#else /* Support for MASA input will be added in the future. */ ivas_error IVAS_REND_FeedInputMasaMetadata( IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ const IVAS_REND_InputId inputId, /* i : ID of the input */ void* TODO ); +#endif -ivas_error IVAS_REND_InitConfig( +ivas_error IVAS_REND_InitConfig( IVAS_REND_HANDLE st, /* i/o: Renderer handle */ bool rendererConfigEnabled /* i : flag indicating if a renderer configuration file was supplied */ ); diff --git a/tests/renderer/constants.py b/tests/renderer/constants.py index cb2d580ec3ecc9342019975a424245feae2653e0..da7302bdc2a4ab95bc507fd32eb32fcdfc49834e 100644 --- a/tests/renderer/constants.py +++ b/tests/renderer/constants.py @@ -241,7 +241,7 @@ FORMAT_TO_CREND_FORMAT = { INPUT_FORMATS_AMBI = ["FOA", "HOA2", "HOA3"] INPUT_FORMATS_MC = ["MONO", "STEREO", "5_1", "5_1_2", "5_1_4", "7_1", "7_1_4"] INPUT_FORMATS_ISM = ["ISM1", "ISM2", "ISM3", "ISM4"] -INPUT_FORMATS_MASA = ["MASA1", "MASA2"] +INPUT_FORMATS_MASA = ["MASA2"] #["MASA1", "MASA2"] # Disable MASA1 tests until MASA1 can be implemented properly """ Non binaural / parametric output formats """ OUTPUT_FORMATS = [ diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index 9660910afc8f4ad057e84279e0ac9172bce3a473..97ccf03c940374d2c9a0deaa6365d83df843bcae 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -156,13 +156,15 @@ def test_ism_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file): """ MASA """ -# # MASA inputs not supported yet -# @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) -# @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) -# def test_masa(test_info, in_fmt, out_fmt): -# # TODO: implement MASA in Python, compare BE -# compare_renderer_vs_pyscripts( test_info, in_fmt, out_fmt, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt] -# ) + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) +def test_masa(test_info, in_fmt, out_fmt): + # # TODO: implement MASA in Python, compare BE + # compare_renderer_vs_pyscripts( test_info, in_fmt, out_fmt, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt] + # ) + run_renderer(in_fmt, out_fmt, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt]) # MASA inputs not supported yet