diff --git a/apps/decoder.c b/apps/decoder.c index 616b36e37e768c8b712013a48ec6dab492622b43..9a4a80312dd4914738e4298a24afb4fb0bd7c7a8 100644 --- a/apps/decoder.c +++ b/apps/decoder.c @@ -106,7 +106,12 @@ typedef struct bool customLsOutputEnabled; char *customLsSetupFilename; int16_t orientation_tracking; +#ifdef NON_DIEGETIC_PAN + int16_t Opt_non_diegetic_pan; + float non_diegetic_pan_gain; +#else float no_diegetic_pan; +#endif bool renderConfigEnabled; char *renderConfigFilename; #ifdef COMPLEXITY_LEVEL_INDICATION @@ -193,7 +198,11 @@ int main( * Open decoder handle *------------------------------------------------------------------------------------------*/ +#ifdef NON_DIEGETIC_PAN + if ( ( error = IVAS_DEC_Open( &hIvasDec, arg.decMode, arg.orientation_tracking ) ) != IVAS_ERR_OK ) +#else if ( ( error = IVAS_DEC_Open( &hIvasDec, arg.decMode, arg.orientation_tracking, arg.no_diegetic_pan ) ) != IVAS_ERR_OK ) +#endif { fprintf( stderr, "Open failed: %s\n", IVAS_DEC_GetErrorMessage( error ) ); goto cleanup; @@ -358,7 +367,11 @@ int main( * Configure the decoder *------------------------------------------------------------------------------------------*/ +#ifdef NON_DIEGETIC_PAN + if ( ( error = IVAS_DEC_Configure( hIvasDec, arg.output_Fs, arg.outputFormat, arg.customLsOutputEnabled, arg.hrtfReaderEnabled, arg.enableHeadRotation, arg.renderConfigEnabled, arg.Opt_non_diegetic_pan, arg.non_diegetic_pan_gain ) ) != IVAS_ERR_OK ) +#else if ( ( error = IVAS_DEC_Configure( hIvasDec, arg.output_Fs, arg.outputFormat, arg.customLsOutputEnabled, arg.hrtfReaderEnabled, arg.enableHeadRotation, arg.renderConfigEnabled ) ) != IVAS_ERR_OK ) +#endif { fprintf( stderr, "\nConfigure failed: %s\n\n", IVAS_DEC_GetErrorMessage( error ) ); goto cleanup; @@ -793,7 +806,12 @@ static bool parseCmdlIVAS_dec( arg->renderConfigFilename = NULL; arg->inputFormat = IVAS_DEC_INPUT_FORMAT_G192; +#ifdef NON_DIEGETIC_PAN + arg->Opt_non_diegetic_pan = 0; + arg->non_diegetic_pan_gain = 0.f; +#else arg->no_diegetic_pan = 0.f; +#endif /*-----------------------------------------------------------------* * Initialization @@ -1019,38 +1037,61 @@ static bool parseCmdlIVAS_dec( } i += 2; } - else if ( strcmp( argv_to_upper, "-NO_DIEGETIC_PAN" ) == 0 ) + else if ( strcmp( argv_to_upper, "-NON_DIEGETIC_PAN" ) == 0 ) { i++; - - if ( argc - i <= 4 || argv[i][0] == '-' ) +#ifdef NON_DIEGETIC_PAN + arg->Opt_non_diegetic_pan = 1; +#else + if ( argc - i <= 4 || ( argv[i][0] == '-' ) ) { fprintf( stderr, "Error: Argument for panning option not specified!\n\n" ); usage_dec(); return false; } - +#endif strncpy( argv_to_upper, argv[i], sizeof( argv_to_upper ) - 1 ); argv_to_upper[sizeof( argv_to_upper ) - 1] = '\0'; +#ifdef NON_DIEGETIC_PAN + to_upper( argv_to_upper ); +#endif if ( ( strcmp( argv_to_upper, "CENTER" ) == 0 ) || ( strchr( argv_to_upper, 'C' ) != NULL ) ) { +#ifdef NON_DIEGETIC_PAN + arg->non_diegetic_pan_gain = 0.f; +#else arg->no_diegetic_pan = 0.f; +#endif } else if ( ( strcmp( argv_to_upper, "LEFT" ) == 0 ) || ( strchr( argv_to_upper, 'L' ) != NULL ) ) { - arg->no_diegetic_pan = -1.f; +#ifdef NON_DIEGETIC_PAN + arg->non_diegetic_pan_gain = 1.f; +#else + arg->no_diegetic_pan = 1.f; +#endif } else if ( ( strcmp( argv_to_upper, "RIGHT" ) == 0 ) || ( strchr( argv_to_upper, 'R' ) != NULL ) ) { - arg->no_diegetic_pan = 1.f; +#ifdef NON_DIEGETIC_PAN + arg->non_diegetic_pan_gain = -1.f; +#else + arg->no_diegetic_pan = -1.f; +#endif } else { +#ifdef NON_DIEGETIC_PAN + arg->non_diegetic_pan_gain = (float) atof( argv_to_upper ) / 90.f; + + if ( arg->non_diegetic_pan_gain > 1.0f || arg->non_diegetic_pan_gain < -1.0f ) +#else arg->no_diegetic_pan = (float) atof( argv_to_upper ); if ( arg->no_diegetic_pan > 1.0f || arg->no_diegetic_pan < -1.0f ) +#endif { - fprintf( stderr, "Error: Incorrect value for panning option argument specified: %s\n\n", argv[i] ); + fprintf( stderr, "Error: Incorrect value for panning gain value specified: %s\n\n", argv[i] ); usage_dec(); return false; } @@ -1104,11 +1145,25 @@ static bool parseCmdlIVAS_dec( arg->customLsSetupFilename = argv[i]; } i++; +#ifdef NON_DIEGETIC_PAN + if ( ( arg->Opt_non_diegetic_pan ) && ( arg->outputFormat != IVAS_DEC_OUTPUT_STEREO ) ) + { + fprintf( stderr, "Error: non-diegetic panning is supported in stereo only\n\n" ); + usage_dec(); + return false; + } +#endif } else { arg->outputFormat = IVAS_DEC_OUTPUT_MONO; arg->decMode = IVAS_DEC_MODE_EVS; +#ifdef NON_DIEGETIC_PAN + if ( ( arg->Opt_non_diegetic_pan ) ) + { + arg->outputFormat = IVAS_DEC_OUTPUT_STEREO; + } +#endif } /*-----------------------------------------------------------------* @@ -1216,8 +1271,8 @@ static void usage_dec( void ) fprintf( stdout, "-rvf File : Reference vector specified by external trajectory file\n" ); fprintf( stdout, " works only in combination with '-otr ref_vec' and 'ref_vec_lev' modes\n" ); fprintf( stdout, "-render_config File : Renderer configuration File\n" ); - fprintf( stdout, "-no_diegetic_pan : panning mono non-diegetic sound to stereo -1<= pan <=1,\n" ); - fprintf( stdout, " left or l or 1->left, right or r or -1->right, center or c or 0->middle\n" ); + fprintf( stdout, "-non_diegetic_pan P : panning mono non-diegetic sound to stereo with paning P, -90<= P <=90,\n" ); + fprintf( stdout, " left or l or 90->left, right or r or -90->right, center or c or 0->middle\n" ); fprintf( stdout, "-q : Quiet mode, no frame counter\n" ); fprintf( stdout, " default is deactivated\n" ); #ifdef DEBUGGING diff --git a/apps/renderer.c b/apps/renderer.c index 9045bbeaa02161270702583fcb114320143104ae..533472b428c854c7a0deba8804e75de4d1c562a4 100644 --- a/apps/renderer.c +++ b/apps/renderer.c @@ -136,7 +136,12 @@ typedef struct char customHrtfFilePath[RENDERER_MAX_CLI_ARG_LENGTH]; char renderConfigFilePath[RENDERER_MAX_CLI_ARG_LENGTH]; int8_t orientationTracking; +#ifdef NON_DIEGETIC_PAN + int16_t nonDiegeticPan; + float nonDiegeticPanGain; +#else float noDiegeticPan; +#endif bool delayCompensationEnabled; bool quietModeEnabled; bool sceneDescriptionInput; @@ -160,7 +165,7 @@ typedef enum CmdLnOptionId_refRotFile, CmdLnOptionId_customHrtfFile, CmdLnOptionId_renderConfigFile, - CmdLnOptionId_noDiegeticPan, + CmdLnOptionId_nonDiegeticPan, CmdLnOptionId_orientationTracking, CmdlnOptionId_lfePosition, CmdlnOptionId_lfeMatrix, @@ -234,10 +239,10 @@ static const CmdLnParser_Option cliOptions[] = { .description = "Binaural renderer configuration file (only for BINAURAL and BINAURAL_ROOM outputs)", }, { - .id = CmdLnOptionId_noDiegeticPan, - .match = "no_diegetic_pan", + .id = CmdLnOptionId_nonDiegeticPan, + .match = "non_diegetic_pan", .matchShort = "ndp", - .description = "Panning mono no diegetic sound to stereo -1<= pan <= 1\nleft or l or 1->left, right or r or -1->right, center or c or 0 ->middle\n(todo: implementation)", + .description = "Panning mono non diegetic sound to stereo -90<= pan <= 90\nleft or l or 90->left, right or r or -90->right, center or c or 0 ->middle\n(todo: implementation)", }, { .id = CmdLnOptionId_orientationTracking, @@ -551,6 +556,21 @@ int main( CmdlnArgs args = parseCmdlnArgs( argc, argv ); +#ifdef NON_DIEGETIC_PAN + if ( args.nonDiegeticPan && !( ( args.inConfig.numAudioObjects == 0 && args.inConfig.multiChannelBuses[0].audioConfig == IVAS_REND_AUDIO_CONFIG_MONO ) || + ( args.inConfig.numAudioObjects > 0 && args.inConfig.audioObjects[0].audioConfig == IVAS_REND_AUDIO_CONFIG_OBJECT && args.inConfig.numAudioObjects == 1 ) ) ) + { + fprintf( stderr, "\ninvalid configuration - non-diegetic panning requires mono or ISM1 input\n" ); + exit( -1 ); + } + + if ( args.nonDiegeticPan && args.outConfig.audioConfig != IVAS_REND_AUDIO_CONFIG_STEREO ) + { + fprintf( stderr, "\ninvalid configuration - non-diegetic panning requires stereo output\n" ); + exit( -1 ); + } +#endif + positionProvider = IsmPositionProvider_open(); convert_backslash( args.inputFilePath ); @@ -665,7 +685,11 @@ int main( IVAS_REND_InputId sbaIds[RENDERER_MAX_SBA_INPUTS] = { 0 }; IVAS_REND_InputId masaIds[RENDERER_MAX_MASA_INPUTS] = { 0 }; +#ifdef NON_DIEGETIC_PAN + if ( ( error = IVAS_REND_Open( &hIvasRend, args.sampleRate, args.outConfig.audioConfig, args.nonDiegeticPan, args.nonDiegeticPanGain ) ) != IVAS_ERR_OK ) +#else if ( ( error = IVAS_REND_Open( &hIvasRend, args.sampleRate, args.outConfig.audioConfig ) ) != IVAS_ERR_OK ) +#endif { fprintf( stderr, "Error opening renderer handle: %s\n", ivas_error_to_string( error ) ); exit( -1 ); @@ -1334,6 +1358,38 @@ static bool parseOutConfig( return true; } +#ifdef NON_DIEGETIC_PAN +static bool parseDiegeticPan( + char *value, + float *nonDiegeticPan ) +{ + to_upper( value ); + + if ( ( strcmp( value, "CENTER" ) == 0 ) || ( strchr( value, 'C' ) != NULL ) ) + { + *nonDiegeticPan = 0.f; + } + else if ( ( strcmp( value, "LEFT" ) == 0 ) || ( strchr( value, 'L' ) != NULL ) ) + { + *nonDiegeticPan = 1.f; + } + else if ( ( strcmp( value, "RIGHT" ) == 0 ) || ( strchr( value, 'R' ) != NULL ) ) + { + *nonDiegeticPan = -1.f; + } + else + { + *nonDiegeticPan = (float) atof( value ) / 90.f; + + if ( *nonDiegeticPan > 1.0f || *nonDiegeticPan < -1.0f ) + { + fprintf( stderr, "Error: Incorrect value for panning option argument specified!\n\n" ); + return false; + } + } + return true; + +#else static bool parseDiegeticPan( char *value, float *noDiegeticPan ) @@ -1363,6 +1419,7 @@ static bool parseDiegeticPan( } } return false; +#endif } static bool parseOrientationTracking( @@ -1615,7 +1672,12 @@ static CmdlnArgs defaultArgs( clearString( args.renderConfigFilePath ); args.orientationTracking = IVAS_ORIENT_TRK_NONE; +#ifdef NON_DIEGETIC_PAN + args.nonDiegeticPan = 0; + args.nonDiegeticPanGain = 0.f; +#else args.noDiegeticPan = 0.0f; +#endif args.delayCompensationEnabled = true; args.quietModeEnabled = false; @@ -1705,13 +1767,20 @@ static void parseOption( assert( numOptionValues == 1 ); strncpy( args->renderConfigFilePath, optionValues[0], RENDERER_MAX_CLI_ARG_LENGTH - 1 ); break; - case CmdLnOptionId_noDiegeticPan: + case CmdLnOptionId_nonDiegeticPan: assert( numOptionValues == 1 ); +#ifdef NON_DIEGETIC_PAN + if ( !parseDiegeticPan( optionValues[0], &args->nonDiegeticPanGain ) ) +#else if ( !parseDiegeticPan( optionValues[0], &args->noDiegeticPan ) ) +#endif { fprintf( stderr, "Unknown option for diegetic panning: %s\n", optionValues[0] ); exit( -1 ); } +#ifdef NON_DIEGETIC_PAN + args->nonDiegeticPan = 1; +#endif break; case CmdLnOptionId_orientationTracking: assert( numOptionValues == 1 ); @@ -1822,6 +1891,9 @@ void getMetadataFromFileReader( objectMetadataBuffer->positions[objIdx].radius = ismMetadata.radius; objectMetadataBuffer->positions[objIdx].yaw = ismMetadata.yaw; objectMetadataBuffer->positions[objIdx].pitch = ismMetadata.pitch; +#ifdef ISM_NON_DIEGETIC_PAN + objectMetadataBuffer->positions[objIdx].non_diegetic_flag = ismMetadata.non_diegetic_flag; +#endif return; } @@ -1879,6 +1951,9 @@ static void IsmPositionProvider_getNextFrame( objectMetadataBuffer->positions[objIdx].radius = 1.0f; objectMetadataBuffer->positions[objIdx].yaw = 0.0f; objectMetadataBuffer->positions[objIdx].pitch = 0.0f; +#ifdef ISM_NON_DIEGETIC_PAN + objectMetadataBuffer->positions[objIdx].non_diegetic_flag = 0; +#endif } /* Wrap azimuth to lie within (-180, 180] range */ @@ -2140,13 +2215,22 @@ static void parseObjectPosition( { char *endptr; int16_t read_values; +#ifdef ISM_NON_DIEGETIC_PAN + float meta_prm[8] = { 0.0f, 0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f }; +#else float meta_prm[7] = { 0.0f, 0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f }; +#endif + readNextMetadataChunk( line, "," ); *positionDuration = (uint16_t) strtol( line, &endptr, 10 ); readNextMetadataChunk( line, "\n" ); +#ifdef ISM_NON_DIEGETIC_PAN + read_values = (int16_t) sscanf( line, "%f,%f,%f,%f,%f,%f,%f,%f", &meta_prm[0], &meta_prm[1], &meta_prm[2], &meta_prm[3], &meta_prm[4], &meta_prm[5], &meta_prm[6], &meta_prm[7] ); +#else read_values = (int16_t) sscanf( line, "%f,%f,%f,%f,%f,%f,%f", &meta_prm[0], &meta_prm[1], &meta_prm[2], &meta_prm[3], &meta_prm[4], &meta_prm[5], &meta_prm[6] ); +#endif if ( read_values < 2 ) { @@ -2159,6 +2243,9 @@ static void parseObjectPosition( position->radius = meta_prm[2]; position->yaw = meta_prm[5]; position->pitch = meta_prm[6]; +#ifdef ISM_NON_DIEGETIC_PAN + position->non_diegetic_flag = (int16_t) meta_prm[7]; +#endif return; } diff --git a/lib_com/cnst.h b/lib_com/cnst.h index 261ef6f907f1777ec47a9c01f3ff7661b5999b80..cd3aa07c48a2d43d95ae7c88e0bb1ee667dacc59 100644 --- a/lib_com/cnst.h +++ b/lib_com/cnst.h @@ -2142,7 +2142,7 @@ enum #define IGF_GRID_LB_SHORT 2 /* constants for IGFSCFDecoder and IGFSCFEncoder */ -#define IGF_CTX_OFFSET 3 /* offset added to make the context values nonnegative, for array indexing */ +#define IGF_CTX_OFFSET 3 /* offset added to make the context values non negative, for array indexing */ #define IGF_CTX_COUNT ( 2 * IGF_CTX_OFFSET + 1 ) /* number of contexts for the AC statistical model */ #define IGF_MIN_ENC_SEPARATE -12 /* minimum residual value coded separately, without escape coding */ #define IGF_MAX_ENC_SEPARATE +12 /* maximum residual value coded separately, without escape coding */ diff --git a/lib_com/common_api_types.h b/lib_com/common_api_types.h index 61c68eae5d5bc8ceeb717b7bd4a7b2cc1112575c..a1ab432ead3a38ca56c0098160a6139ada36c7ed 100644 --- a/lib_com/common_api_types.h +++ b/lib_com/common_api_types.h @@ -80,6 +80,9 @@ typedef struct _IVAS_ISM_METADATA float gainFactor; float yaw; float pitch; +#ifdef ISM_NON_DIEGETIC_PAN + int16_t non_diegetic_flag; +#endif } IVAS_ISM_METADATA; typedef struct @@ -118,6 +121,9 @@ typedef struct float radius; float yaw; float pitch; +#ifdef ISM_NON_DIEGETIC_PAN + int16_t non_diegetic_flag; +#endif } IVAS_REND_AudioObjectPosition; typedef struct _IVAS_ROOM_ACOUSTICS_CONFIG diff --git a/lib_com/hq_tools.c b/lib_com/hq_tools.c index 1d89d0314597493852e85af60f0e57247410d33c..0008cb957ce7b5a85d8c0b50478b02995f5f7eeb 100644 --- a/lib_com/hq_tools.c +++ b/lib_com/hq_tools.c @@ -1290,7 +1290,6 @@ int16_t calc_nor_delta_hf( ynrm[i] += delta; add_bits_denv += bitsforDelta; - temp_num++; } } diff --git a/lib_com/ivas_cnst.h b/lib_com/ivas_cnst.h index 859968565eaa6ea27e7e8c2f025a0ce851b6c558..4a260abe2e6fa85ae0db69d4972edfb89cd4f022 100755 --- a/lib_com/ivas_cnst.h +++ b/lib_com/ivas_cnst.h @@ -156,7 +156,12 @@ typedef enum RENDERER_MCMASA_MONO_STEREO, RENDERER_PARAM_ISM, RENDERER_BINAURAL_MIXER_CONV, +#if defined NON_DIEGETIC_PAN || defined ISM_NON_DIEGETIC_PAN + RENDERER_BINAURAL_MIXER_CONV_ROOM, + RENDERER_NON_DIEGETIC_DOWNMIX +#else RENDERER_BINAURAL_MIXER_CONV_ROOM +#endif } RENDERER_TYPE; @@ -170,6 +175,9 @@ typedef enum #define HEAD_ROTATION_HOA_ORDER 3 /* HOA 3rd order */ #define MAX_CICP_CHANNELS 16 /* max channels for loudspeaker layouts (16 for custom layouts)*/ #define MAX_OUTPUT_CHANNELS 16 /* Maximum number of output channels (HOA 3rd order) */ +#ifdef NON_DIEGETIC_PAN +#define MAX_OUTPUT_CHANNELS_IN_DIEGETIC_PAN 2 /* Maximum number of output channels with non diegetic panning */ +#endif #define BINAURAL_CHANNELS 2 /* number of channels for binaural output configuration */ #define CPE_CHANNELS 2 /* number of CPE (stereo) channels */ @@ -332,6 +340,9 @@ typedef enum #define ISM_RADIUS_MIN 0.0f #define ISM_RADIUS_DELTA 0.25f /* Max radius = (2^ISM_RADIUS_NBITS-1)*0.25 = 15.75 */ #define ISM_EXTENDED_METADATA_BRATE IVAS_64k +#ifdef ISM_NON_DIEGETIC_PAN +#define ISM_METADATA_IS_NDP_BITS 1 +#endif #define ISM_EXTENDED_METADATA_BITS 1 #define ISM_METADATA_RS_MAX_FRAMES 5 /* Number of frames with opposite extended metadata flags before switching */ @@ -373,6 +384,9 @@ enum { IND_ISM_NUM_OBJECTS, IND_ISM_EXTENDED_FLAG = IND_ISM_NUM_OBJECTS + MAX_NUM_OBJECTS, +#ifdef ISM_NON_DIEGETIC_PAN + IND_ISM_EXTENDED_NDP_FLAG, +#endif IND_ISM_METADATA_FLAG, IND_ISM_VAD_FLAG = IND_ISM_METADATA_FLAG + MAX_NUM_OBJECTS, IND_ISM_NOISY_SPEECH_FLAG = IND_ISM_VAD_FLAG + MAX_NUM_OBJECTS, @@ -381,6 +395,9 @@ enum /* ------------- loop for objects -------------- */ TAG_ISM_LOOP_START = IND_ISM_DTX_COH_SCA + MAX_NUM_OBJECTS, +#ifdef ISM_NON_DIEGETIC_PAN + IND_ISM_NDP_FLAG = TAG_ISM_LOOP_START, +#endif IND_ISM_AZIMUTH_DIFF_FLAG = TAG_ISM_LOOP_START, IND_ISM_AZIMUTH = TAG_ISM_LOOP_START, IND_ISM_ELEVATION_DIFF_FLAG = TAG_ISM_LOOP_START, @@ -1572,7 +1589,12 @@ typedef enum typedef enum { TDREND_PLAYSTATUS_INITIAL, +#ifdef ISM_NON_DIEGETIC_PAN + TDREND_PLAYSTATUS_PLAYING, + TDREND_PLAYSTATUS_PLAYING_NON_DIEGETIC +#else TDREND_PLAYSTATUS_PLAYING +#endif } TDREND_PlayStatus_t; typedef enum diff --git a/lib_com/ivas_ism_com.c b/lib_com/ivas_ism_com.c index 38a34a15bdbe3e9a8ec053cd065508c7aaf6764a..d14b252fd87b163b23d6e27062b53bed5cf948fc 100644 --- a/lib_com/ivas_ism_com.c +++ b/lib_com/ivas_ism_com.c @@ -88,10 +88,13 @@ static void bitbudget_to_brate( *-------------------------------------------------------------------*/ ivas_error ivas_ism_config( - const int32_t ism_total_brate, /* i : ISM total bitrate */ - const int16_t nchan_transport, /* i : number of transport channels */ - const int16_t nchan_ism, /* i : number of objects */ - ISM_METADATA_HANDLE hIsmMeta[], /* i/o: ISM metadata handles */ + const int32_t ism_total_brate, /* i : ISM total bitrate */ + const int16_t nchan_transport, /* i : number of transport channels */ + const int16_t nchan_ism, /* i : number of objects */ + ISM_METADATA_HANDLE hIsmMeta[], /* i/o: ISM metadata handles */ +#ifdef ISM_NON_DIEGETIC_PAN + const int16_t ism_extended_metadata_flag, /* i : extended metadata flag */ +#endif const int16_t localVAD[MAX_NUM_OBJECTS], /* i : local VAD flag */ const int16_t ism_imp[], /* i : ISM importance flags */ int32_t element_brate[], /* o : element bitrate per object */ @@ -135,6 +138,13 @@ ivas_error ivas_ism_config( if ( ism_total_brate >= ISM_EXTENDED_METADATA_BRATE ) { nb_bits_metadata[0] += ISM_EXTENDED_METADATA_BITS; + +#ifdef ISM_NON_DIEGETIC_PAN + if ( ism_extended_metadata_flag ) + { + nb_bits_metadata[0] += ISM_METADATA_IS_NDP_BITS; + } +#endif } nb_bits_metadata[0] += n_ISms * ISM_METADATA_FLAG_BITS + nchan_ism; @@ -337,6 +347,10 @@ void ivas_ism_reset_metadata( hIsmMeta->yaw = 0.0f; hIsmMeta->pitch = 0.0f; hIsmMeta->radius = 1.0f; +#ifdef ISM_NON_DIEGETIC_PAN + hIsmMeta->ism_metadata_flag = 0; + hIsmMeta->non_diegetic_flag = 0; +#endif return; } @@ -352,7 +366,9 @@ void ivas_ism_reset_metadata_API( ISM_METADATA_HANDLE hIsmMeta /* i/o: ISM metadata handle */ ) { +#ifndef ISM_NON_DIEGETIC_PAN hIsmMeta->ism_metadata_flag = 0; +#endif ivas_ism_reset_metadata( hIsmMeta ); return; @@ -454,9 +470,8 @@ float ism_dequant_meta( * ---------------------------------------------------------------*/ void ivas_param_ism_config( - PARAM_ISM_CONFIG_HANDLE hParamIsm /* i/o: IVAS Param ISM Config Structure */ - , - const int16_t nchan_obj /* i : number of ISM channels */ + PARAM_ISM_CONFIG_HANDLE hParamIsm, /* i/o: IVAS Param ISM Config Structure */ + const int16_t nchan_obj /* i : number of ISM channels */ ) { int16_t i; diff --git a/lib_com/ivas_prot.h b/lib_com/ivas_prot.h index 5348abe1d78fd9f1478a6a03972d0be3af98a6d2..14a11b105ed8e94cde1a8c5226a57c06dcbeb2e6 100644 --- a/lib_com/ivas_prot.h +++ b/lib_com/ivas_prot.h @@ -746,6 +746,9 @@ ivas_error ivas_ism_config( const int16_t nchan_transport, /* i : number of transport channels */ const int16_t nchan_ism, /* i : number of objects */ ISM_METADATA_HANDLE hIsmMeta[], /* i/o: ISM metadata handles */ +#ifdef ISM_NON_DIEGETIC_PAN + const int16_t ism_extended_metadata_flag, /* i : extended metadata flag */ +#endif const int16_t localVAD[MAX_NUM_OBJECTS], /* i : local VAD flag */ const int16_t ism_imp[], /* i : ISM importance flags */ int32_t element_brate[], /* o : element bitrate per object */ @@ -786,7 +789,12 @@ ivas_error ivas_set_ism_metadata( const float elevation, /* i : elevation value */ const float radius_meta, /* i : radius */ const float yaw, /* i : yaw */ +#ifdef ISM_NON_DIEGETIC_PAN + const float pitch, /* i : pitch */ + const int16_t non_diegetic_flag /* i : non-diegetic object flag */ +#else const float pitch /* i : pitch */ +#endif ); ivas_error ivas_ism_metadata_enc_create( diff --git a/lib_com/ivas_stat_com.h b/lib_com/ivas_stat_com.h index 965bd1f56ba1e963c792cc207080d4e4d953fde7..a63b2ebd6b2501886df6557d481190083c845f71 100644 --- a/lib_com/ivas_stat_com.h +++ b/lib_com/ivas_stat_com.h @@ -43,19 +43,24 @@ /*----------------------------------------------------------------------------------* * Declaration of ISM common (encoder & decoder) structure *----------------------------------------------------------------------------------*/ + typedef struct { int16_t last_angle1_idx; /* last frame index of coded azimuth/yaw */ int16_t angle1_diff_cnt; /* FEC counter of consecutive differentially azimuth/yaw coded frames */ int16_t last_angle2_idx; /* last frame index of coded elevation/pitch */ int16_t angle2_diff_cnt; /* FEC counter of consecutive differentially elevation/pitch coded frames */ + } ISM_METADATA_ANGLE, *ISM_METADATA_ANGLE_HANDLE; + /* ISM metadata handle (storage for one frame of read ISM metadata) */ typedef struct { int16_t ism_metadata_flag; /* flag whether metadata are coded in particular frame of particular object */ int16_t last_ism_metadata_flag; /* last frame ism_metadata_flag */ - +#ifdef ISM_NON_DIEGETIC_PAN + int16_t non_diegetic_flag; /* Non-diegetic (non-headtracked) object flag */ +#endif float azimuth; /* azimuth value read from the input metadata file */ float elevation; /* azimuth value read from the input metadata file */ float radius; @@ -78,6 +83,7 @@ typedef struct #ifdef FIX_435_ISM_MERGE_BUG float last_true_radius; /* last true Q radius value */ #endif + } ISM_METADATA_FRAME, *ISM_METADATA_HANDLE; diff --git a/lib_com/options.h b/lib_com/options.h index e68aca0ef1b9d22f274a6307a416732398b1f9bd..655c16b308835db9191cd050cc5014975cce6d5b 100755 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -137,6 +137,8 @@ #define DISABLE_ADAP_RES_COD_TMP /* temporary fix for IVAS-403, disables adaptive residual coding */ /*#define FIX_I4_OL_PITCH*/ /* fix open-loop pitch used for EVS core switching */ +#define NON_DIEGETIC_PAN /* Orange: Contribution 34: non diegetic panning in decoder */ +#define ISM_NON_DIEGETIC_PAN /* Orange: Contribution 34: non diegetic panning for ism objects */ #define FIX_103_RA_PARAMS_PARAM_BIN_REND /* Issue 103: Digest room acoustics parameters for Parametric Binaural Renderer*/ /*#define SBA_HPF_TUNING_DEC*/ #define SMOOTH_WITH_TRANS_DET diff --git a/lib_dec/ivas_dec.c b/lib_dec/ivas_dec.c index d33dcbe1a187e492f039e742d77375afa2b43510..b60be084f9797bb74b6e1f9124c1d126c1609cd0 100644 --- a/lib_dec/ivas_dec.c +++ b/lib_dec/ivas_dec.c @@ -64,6 +64,9 @@ ivas_error ivas_dec( int16_t nb_bits_metadata[MAX_SCE]; int32_t output_Fs, ivas_total_brate; AUDIO_CONFIG output_config; +#ifdef NON_DIEGETIC_PAN + float pan_left, pan_right; +#endif ivas_error error; error = IVAS_ERR_OK; @@ -191,6 +194,15 @@ ivas_error ivas_dec( { ivas_mono_downmix_render_passive( st_ivas, output, output_frame ); } +#ifdef NON_DIEGETIC_PAN + else if ( st_ivas->renderer_type == RENDERER_NON_DIEGETIC_DOWNMIX ) + { + pan_left = ( st_ivas->hDecoderConfig->non_diegetic_pan_gain + 1.f ) * 0.5f; + pan_right = 1.f - pan_left; + v_multc( output[0], pan_right, output[1], output_frame ); + v_multc( output[0], pan_left, output[0], output_frame ); + } +#endif else if ( st_ivas->renderer_type == RENDERER_PARAM_ISM || st_ivas->renderer_type == RENDERER_SBA_LINEAR_ENC ) { ivas_param_ism_dec( st_ivas, output ); @@ -209,6 +221,15 @@ ivas_error ivas_dec( { ivas_mono_downmix_render_passive( st_ivas, output, output_frame ); } +#ifdef NON_DIEGETIC_PAN + else if ( st_ivas->renderer_type == RENDERER_NON_DIEGETIC_DOWNMIX ) + { + pan_left = ( st_ivas->hDecoderConfig->non_diegetic_pan_gain + 1.f ) * 0.5f; + pan_right = 1.f - pan_left; + v_multc( output[0], pan_right, output[1], output_frame ); + v_multc( output[0], pan_left, output[0], output_frame ); + } +#endif else if ( st_ivas->renderer_type == RENDERER_TD_PANNING || st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV_ROOM ) { /* Convert to CICPxx; used also for ISM->CICP19->binaural_room rendering */ diff --git a/lib_dec/ivas_init_dec.c b/lib_dec/ivas_init_dec.c index 3f777334da3e555114e345dbfbf6329b07585587..271b756f7a6658890cf47a5c3b636573f4a53497 100644 --- a/lib_dec/ivas_init_dec.c +++ b/lib_dec/ivas_init_dec.c @@ -1353,6 +1353,9 @@ ivas_error ivas_init_decoder( if ( st_ivas->ivas_format == ISM_FORMAT && st_ivas->ism_mode == ISM_MODE_DISC && ( st_ivas->renderer_type == RENDERER_TD_PANNING || +#if defined NON_DIEGETIC_PAN || defined ISM_NON_DIEGETIC_PAN + st_ivas->renderer_type == RENDERER_NON_DIEGETIC_DOWNMIX || +#endif st_ivas->renderer_type == RENDERER_SBA_LINEAR_ENC || st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV || st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV_ROOM || @@ -1899,6 +1902,9 @@ void ivas_init_dec_get_num_cldfb_instances( *numCldfbAnalyses += 2; } break; +#ifdef NON_DIEGETIC_PAN + case RENDERER_NON_DIEGETIC_DOWNMIX: +#endif case RENDERER_MONO_DOWNMIX: if ( st_ivas->ivas_format == ISM_FORMAT ) { @@ -2067,6 +2073,14 @@ static ivas_error doSanityChecks_IVAS( assert( st_ivas->ivas_format != UNDEFINED_FORMAT && "\n IVAS format undefined" ); assert( st_ivas->ivas_format != MONO_FORMAT && "\n Wrong IVAS format: MONO" ); + /* Verify output configuration compatible with non-diegetic panning */ +#ifdef NON_DIEGETIC_PAN + if ( st_ivas->hDecoderConfig->Opt_non_diegetic_pan && ( st_ivas->ivas_format != MONO_FORMAT ) && ( st_ivas->transport_config != AUDIO_CONFIG_ISM1 ) ) + { + return IVAS_ERROR( IVAS_ERR_INVALID_OUTPUT_FORMAT, "Error: Non-diegetic panning not supported in this IVAS format" ); + } +#endif + /* Verify stereo output configuration */ if ( st_ivas->ivas_format == STEREO_FORMAT ) { diff --git a/lib_dec/ivas_ism_dec.c b/lib_dec/ivas_ism_dec.c index dda939529d0597262156af814db645645614f574..e4eac7cb610b582f943a965feef67cf9ddfd03d9 100644 --- a/lib_dec/ivas_ism_dec.c +++ b/lib_dec/ivas_ism_dec.c @@ -72,7 +72,11 @@ static ivas_error ivas_ism_bitrate_switching( st_ivas->ism_mode = ism_mode; #endif +#ifdef ISM_NON_DIEGETIC_PAN + if ( ( error = ivas_ism_config( st_ivas->hDecoderConfig->ivas_total_brate, st_ivas->nchan_transport, st_ivas->nchan_ism, NULL, 0, NULL, NULL, element_brate_tmp, NULL, NULL ) ) != IVAS_ERR_OK ) +#else if ( ( error = ivas_ism_config( st_ivas->hDecoderConfig->ivas_total_brate, st_ivas->nchan_transport, st_ivas->nchan_ism, NULL, NULL, NULL, element_brate_tmp, NULL, NULL ) ) != IVAS_ERR_OK ) +#endif { return error; } diff --git a/lib_dec/ivas_ism_metadata_dec.c b/lib_dec/ivas_ism_metadata_dec.c index 7a61424a1fd3d73b6d5e6b51aecabbcf94993fb1..b7bf89e026c874793b942f96e2e0b8f798cc0591 100644 --- a/lib_dec/ivas_ism_metadata_dec.c +++ b/lib_dec/ivas_ism_metadata_dec.c @@ -48,7 +48,12 @@ * Local functions *-----------------------------------------------------------------------*/ +#ifdef ISM_NON_DIEGETIC_PAN +static void decode_angle_indices( DEC_CORE_HANDLE st0, ISM_METADATA_ANGLE_HANDLE angle, const int16_t non_diegetic_flag, int16_t *flag_abs_azimuth ); +#else static void decode_angle_indices( DEC_CORE_HANDLE st0, ISM_METADATA_ANGLE_HANDLE angle, int16_t *flag_abs_azimuth ); +#endif + static int16_t decode_radius( DEC_CORE_HANDLE st0, int16_t *last_radius_idx, int16_t *flag_abs_radius ); @@ -156,6 +161,9 @@ ivas_error ivas_ism_metadata_dec( int32_t element_brate[MAX_NUM_OBJECTS], total_brate[MAX_NUM_OBJECTS]; DEC_CORE_HANDLE st0; int16_t ism_extmeta_bitstream; +#ifdef ISM_NON_DIEGETIC_PAN + int16_t non_diegetic_flag_global; +#endif float yaw, pitch, radius; int16_t flag_abs_radius; int16_t flag_abs_orientation; @@ -185,6 +193,9 @@ ivas_error ivas_ism_metadata_dec( next_bit_pos_orig = st0->next_bit_pos; st0->next_bit_pos = 0; ism_extmeta_bitstream = 0; +#ifdef ISM_NON_DIEGETIC_PAN + non_diegetic_flag_global = 0; +#endif /* reverse the bitstream for easier reading of indices */ for ( i = 0; i < min( MAX_BITS_METADATA, last_bit_pos ); i++ ) @@ -224,6 +235,14 @@ ivas_error ivas_ism_metadata_dec( if ( ism_total_brate >= ISM_EXTENDED_METADATA_BRATE ) { ism_extmeta_bitstream = get_next_indice( st0, ISM_EXTENDED_METADATA_BITS ); + +#ifdef ISM_NON_DIEGETIC_PAN + /* read global non-diegetic object flag */ + if ( ism_extmeta_bitstream ) + { + non_diegetic_flag_global = get_next_indice( st0, ISM_METADATA_IS_NDP_BITS ); + } +#endif } /* Apply hysteresis in case rate switching causes fluctuation in presence of extended metadata */ if ( *ism_extmeta_active == -1 || *ism_extmeta_active == ism_extmeta_bitstream ) /* If first frame or bitstream matches internal state */ @@ -304,53 +323,91 @@ ivas_error ivas_ism_metadata_dec( if ( hIsmMeta[ch]->ism_metadata_flag ) { - decode_angle_indices( st0, &( hIsmMetaData->position_angle ), &flag_abs_position ); - idx_angle1 = hIsmMetaData->position_angle.last_angle1_idx; - idx_angle2 = hIsmMetaData->position_angle.last_angle2_idx; - /* Azimuth/Elevation dequantization */ - if ( ism_mode == ISM_MODE_PARAM ) +#ifdef ISM_NON_DIEGETIC_PAN + if ( non_diegetic_flag_global ) { - hParamIsm->azi_index[ch] = idx_angle1; - hParamIsm->ele_index[ch] = idx_angle2; + /* read non-diegetic flag for each object */ + hIsmMetaData->non_diegetic_flag = get_next_indice( st0, ISM_METADATA_IS_NDP_BITS ); } - else /* ISM_MODE_DISC */ + else { + hIsmMetaData->non_diegetic_flag = 0; + } + + if ( hIsmMetaData->non_diegetic_flag ) + { + /* Panning gain decoding */ + decode_angle_indices( st0, &( hIsmMetaData->position_angle ), hIsmMetaData->non_diegetic_flag, &flag_abs_position ); + idx_angle1 = hIsmMetaData->position_angle.last_angle1_idx; + idx_angle2 = 1 << ( ISM_ELEVATION_NBITS - 1 ); + + /* Panning gain dequantization */ hIsmMetaData->azimuth = ism_dequant_meta( idx_angle1, ism_azimuth_borders, ISM_Q_STEP, ISM_Q_STEP_BORDER, 1 << ISM_AZIMUTH_NBITS ); - hIsmMetaData->elevation = ism_dequant_meta( idx_angle2, ism_elevation_borders, ISM_Q_STEP, ISM_Q_STEP_BORDER, 1 << ISM_ELEVATION_NBITS ); + hIsmMetaData->elevation = 0.0f; - /* radius/raw/pitch dequantization */ - if ( ism_extmeta_bitstream ) + /* save for smoothing metadata evolution */ + hIsmMetaData->last_true_azimuth = hIsmMetaData->azimuth; + hIsmMetaData->last_true_elevation = hIsmMetaData->elevation; + } + else + { + decode_angle_indices( st0, &( hIsmMetaData->position_angle ), hIsmMeta[ch]->non_diegetic_flag, &flag_abs_position ); +#else + decode_angle_indices( st0, &( hIsmMetaData->position_angle ), &flag_abs_position ); +#endif + idx_angle1 = hIsmMetaData->position_angle.last_angle1_idx; + idx_angle2 = hIsmMetaData->position_angle.last_angle2_idx; + + /* Azimuth/Elevation dequantization */ + if ( ism_mode == ISM_MODE_PARAM ) { - decode_angle_indices( st0, &( hIsmMetaData->orientation_angle ), &flag_abs_orientation ); - idx_angle1 = hIsmMetaData->orientation_angle.last_angle1_idx; - idx_angle2 = hIsmMetaData->orientation_angle.last_angle2_idx; - yaw = ism_dequant_meta( idx_angle1, ism_azimuth_borders, ISM_Q_STEP, ISM_Q_STEP_BORDER, 1 << ISM_AZIMUTH_NBITS ); - pitch = ism_dequant_meta( idx_angle2, ism_elevation_borders, ISM_Q_STEP, ISM_Q_STEP_BORDER, 1 << ISM_ELEVATION_NBITS ); - - idx_radius = decode_radius( st0, &hIsmMetaData->last_radius_idx, &flag_abs_radius ); - radius = usdequant( idx_radius, ISM_RADIUS_MIN, ISM_RADIUS_DELTA ); - if ( *ism_extmeta_active == 1 ) - { - hIsmMetaData->yaw = yaw; - hIsmMetaData->pitch = pitch; - hIsmMetaData->radius = radius; - } + hParamIsm->azi_index[ch] = idx_angle1; + hParamIsm->ele_index[ch] = idx_angle2; } - else + else /* ISM_MODE_DISC */ { - if ( *ism_extmeta_active == 0 ) + hIsmMetaData->azimuth = ism_dequant_meta( idx_angle1, ism_azimuth_borders, ISM_Q_STEP, ISM_Q_STEP_BORDER, 1 << ISM_AZIMUTH_NBITS ); + hIsmMetaData->elevation = ism_dequant_meta( idx_angle2, ism_elevation_borders, ISM_Q_STEP, ISM_Q_STEP_BORDER, 1 << ISM_ELEVATION_NBITS ); + + /* radius/raw/pitch dequantization */ + if ( ism_extmeta_bitstream ) { - hIsmMetaData->yaw = 0.0f; - hIsmMetaData->pitch = 0.0f; - hIsmMetaData->radius = 1.0f; +#ifdef ISM_NON_DIEGETIC_PAN + decode_angle_indices( st0, &( hIsmMetaData->orientation_angle ), hIsmMeta[ch]->non_diegetic_flag, &flag_abs_orientation ); +#else + decode_angle_indices( st0, &( hIsmMetaData->orientation_angle ), &flag_abs_orientation ); +#endif + idx_angle1 = hIsmMetaData->orientation_angle.last_angle1_idx; + idx_angle2 = hIsmMetaData->orientation_angle.last_angle2_idx; + yaw = ism_dequant_meta( idx_angle1, ism_azimuth_borders, ISM_Q_STEP, ISM_Q_STEP_BORDER, 1 << ISM_AZIMUTH_NBITS ); + pitch = ism_dequant_meta( idx_angle2, ism_elevation_borders, ISM_Q_STEP, ISM_Q_STEP_BORDER, 1 << ISM_ELEVATION_NBITS ); + + idx_radius = decode_radius( st0, &hIsmMetaData->last_radius_idx, &flag_abs_radius ); + radius = usdequant( idx_radius, ISM_RADIUS_MIN, ISM_RADIUS_DELTA ); + if ( *ism_extmeta_active == 1 ) + { + hIsmMetaData->yaw = yaw; + hIsmMetaData->pitch = pitch; + hIsmMetaData->radius = radius; + } + } + else + { + if ( *ism_extmeta_active == 0 ) + { + hIsmMetaData->yaw = 0.0f; + hIsmMetaData->pitch = 0.0f; + hIsmMetaData->radius = 1.0f; + } } } + /* save for smoothing metadata evolution */ + hIsmMetaData->last_true_azimuth = hIsmMetaData->azimuth; + hIsmMetaData->last_true_elevation = hIsmMetaData->elevation; } - /* save for smoothing metadata evolution */ - hIsmMetaData->last_true_azimuth = hIsmMetaData->azimuth; - hIsmMetaData->last_true_elevation = hIsmMetaData->elevation; +#ifdef ISM_NON_DIEGETIC_PAN } - +#endif /* save number of metadata bits read */ if ( ism_mode == ISM_MODE_DISC ) { @@ -442,7 +499,11 @@ ivas_error ivas_ism_metadata_dec( if ( !bfi ) { +#ifdef ISM_NON_DIEGETIC_PAN + if ( ( error = ivas_ism_config( ism_total_brate, *nchan_transport, nchan_ism, hIsmMeta, ism_extmeta_bitstream, localVAD, ism_imp, element_brate, total_brate, nb_bits_metadata ) ) != IVAS_ERR_OK ) +#else if ( ( error = ivas_ism_config( ism_total_brate, *nchan_transport, nchan_ism, hIsmMeta, localVAD, ism_imp, element_brate, total_brate, nb_bits_metadata ) ) != IVAS_ERR_OK ) +#endif { return error; } @@ -552,7 +613,11 @@ ivas_error ivas_ism_metadata_dec_create( ivas_ism_reset_metadata( st_ivas->hIsmMetaData[ch] ); } +#ifdef ISM_NON_DIEGETIC_PAN + if ( ( error = ivas_ism_config( st_ivas->hDecoderConfig->ivas_total_brate, n_ISms, n_ISms, NULL, 0, NULL, NULL, element_brate_tmp, NULL, NULL ) ) != IVAS_ERR_OK ) +#else if ( ( error = ivas_ism_config( st_ivas->hDecoderConfig->ivas_total_brate, n_ISms, n_ISms, NULL, NULL, NULL, element_brate_tmp, NULL, NULL ) ) != IVAS_ERR_OK ) +#endif { return error; } @@ -572,7 +637,10 @@ ivas_error ivas_ism_metadata_dec_create( static void decode_angle_indices( DEC_CORE_HANDLE st0, /* i/o: bitstream handle */ ISM_METADATA_ANGLE_HANDLE angle, /* i/o: angle handle */ - int16_t *flag_abs_angle1 /* o : Azimuth/yaw encoding mode */ +#ifdef ISM_NON_DIEGETIC_PAN + const int16_t non_diegetic_flag, /* i : Non diegetic flag */ +#endif + int16_t *flag_abs_angle1 /* o : Azimuth/yaw encoding mode */ ) { int16_t idx_angle1, nbits_diff_angle1, diff, sgn; @@ -650,60 +718,75 @@ static void decode_angle_indices( * Elevation/pitch decoding and dequantization *----------------------------------------------------------------*/ - /* Decode elevation/pitch index */ - if ( *flag_abs_angle1 == 0 && get_next_indice( st0, 1 ) == 1 ) /* elevation_abs_flag */ - { - idx_angle2 = get_next_indice( st0, ISM_ELEVATION_NBITS ); - } - else +#ifdef ISM_NON_DIEGETIC_PAN + if ( non_diegetic_flag == 0 ) { - diff = 0; - sgn = 1; - - if ( get_next_indice( st0, 1 ) == 0 ) +#endif + /* Decode elevation/pitch index */ + if ( *flag_abs_angle1 == 0 && get_next_indice( st0, 1 ) == 1 ) /* elevation_abs_flag */ { - nbits_diff_angle2 = 1; + idx_angle2 = get_next_indice( st0, ISM_ELEVATION_NBITS ); } else { - nbits_diff_angle2 = 1; + diff = 0; + sgn = 1; - if ( get_next_indice( st0, 1 ) == 1 ) /* negative sign */ + if ( get_next_indice( st0, 1 ) == 0 ) { - sgn = -1; + nbits_diff_angle2 = 1; } + else + { + nbits_diff_angle2 = 1; - nbits_diff_angle2++; + if ( get_next_indice( st0, 1 ) == 1 ) /* negative sign */ + { + sgn = -1; + } - /* read until the stop bit */ - while ( ( nbits_diff_angle2 < ISM_ELEVATION_NBITS ) && ( get_next_indice( st0, 1 ) == 1 ) ) - { - diff++; nbits_diff_angle2++; - } - if ( nbits_diff_angle2 < ISM_ELEVATION_NBITS ) - { - /* count stop bit */ - nbits_diff_angle2++; + /* read until the stop bit */ + while ( ( nbits_diff_angle2 < ISM_ELEVATION_NBITS ) && ( get_next_indice( st0, 1 ) == 1 ) ) + { + diff++; + nbits_diff_angle2++; + } + + if ( nbits_diff_angle2 < ISM_ELEVATION_NBITS ) + { + /* count stop bit */ + nbits_diff_angle2++; + } } + + idx_angle2 = angle->last_angle2_idx + sgn * diff; } - idx_angle2 = angle->last_angle2_idx + sgn * diff; - } + /* sanity check in case of FER or BER */ + if ( idx_angle2 < 0 || idx_angle2 > ( 1 << ISM_ELEVATION_NBITS ) - 1 ) + { + idx_angle2 = angle->last_angle2_idx; + } - /* sanity check in case of FER or BER */ - if ( idx_angle2 < 0 || idx_angle2 > ( 1 << ISM_ELEVATION_NBITS ) - 1 ) +#ifdef ISM_NON_DIEGETIC_PAN + } + else { - idx_angle2 = angle->last_angle2_idx; + idx_angle2 = angle->last_angle2_idx; /* second MD parameter is not transmitted for non-diegetic object */ } +#endif /*----------------------------------------------------------------* * Final updates *----------------------------------------------------------------*/ + angle->last_angle2_idx = idx_angle2; angle->last_angle1_idx = idx_angle1; +#ifndef ISM_NON_DIEGETIC_PAN angle->last_angle2_idx = idx_angle2; +#endif return; } @@ -912,7 +995,6 @@ void ivas_ism_metadata_sid_dec( for ( ch = 0; ch < nchan_ism; ch++ ) { hIsmMetaData = hIsmMeta[ch]; - if ( md_diff_flag[ch] == 1 ) { /* Azimuth decoding */ diff --git a/lib_dec/ivas_ism_renderer.c b/lib_dec/ivas_ism_renderer.c index 1e60143cb43e0249d3d146fbf6120be3de2d9656..06f5f18467d22b9aab921d93dee89494702a1db5 100644 --- a/lib_dec/ivas_ism_renderer.c +++ b/lib_dec/ivas_ism_renderer.c @@ -149,7 +149,11 @@ void ivas_ism_render( else { /* Head rotation: rotate the object positions depending the head's orientation */ +#ifdef ISM_NON_DIEGETIC_PAN + if ( st_ivas->hHeadTrackData != NULL && st_ivas->hHeadTrackData->num_quaternions >= 0 && !st_ivas->hIsmMetaData[i]->non_diegetic_flag ) +#else if ( st_ivas->hHeadTrackData != NULL && st_ivas->hHeadTrackData->num_quaternions >= 0 ) +#endif { rotateAziEle( st_ivas->hIsmMetaData[i]->azimuth, st_ivas->hIsmMetaData[i]->elevation, &azimuth, &elevation, Rmat, st_ivas->hIntSetup.is_planar_setup ); } diff --git a/lib_dec/ivas_output_config.c b/lib_dec/ivas_output_config.c index f0d9ad84c315c67af470a8275616b072ec4222a1..02cab10026fdc8f41722b058b4429401e0287f8c 100644 --- a/lib_dec/ivas_output_config.c +++ b/lib_dec/ivas_output_config.c @@ -276,6 +276,15 @@ void ivas_renderer_select( * Non-binaural rendering configurations *-----------------------------------------------------------------*/ +#ifdef NON_DIEGETIC_PAN + else if ( st_ivas->ivas_format == MONO_FORMAT ) + { + if ( output_config == AUDIO_CONFIG_STEREO ) + { + *renderer_type = RENDERER_NON_DIEGETIC_DOWNMIX; + } + } +#endif else if ( st_ivas->ivas_format == STEREO_FORMAT ) { if ( output_config != AUDIO_CONFIG_STEREO && output_config != AUDIO_CONFIG_MONO ) @@ -285,39 +294,50 @@ void ivas_renderer_select( } else if ( st_ivas->ivas_format == ISM_FORMAT ) { - if ( st_ivas->ism_mode == ISM_MODE_PARAM ) +#ifdef NON_DIEGETIC_PAN + if ( ( output_config == AUDIO_CONFIG_STEREO ) && ( st_ivas->hDecoderConfig->Opt_non_diegetic_pan ) ) { - *renderer_type = RENDERER_PARAM_ISM; - if ( output_config == AUDIO_CONFIG_MONO ) - { - *renderer_type = RENDERER_MONO_DOWNMIX; - } - else if ( output_config == AUDIO_CONFIG_STEREO ) - { - *renderer_type = RENDERER_DISABLE; - } - else if ( output_config == AUDIO_CONFIG_FOA || output_config == AUDIO_CONFIG_HOA2 || output_config == AUDIO_CONFIG_HOA3 ) - { - *renderer_type = RENDERER_SBA_LINEAR_ENC; - *internal_config = AUDIO_CONFIG_7_1_4; - } + *renderer_type = RENDERER_NON_DIEGETIC_DOWNMIX; } - else /* ISM_MODE_DISC */ + else { - *renderer_type = RENDERER_TD_PANNING; - if ( output_config == AUDIO_CONFIG_MONO ) - { - *renderer_type = RENDERER_MONO_DOWNMIX; - } - else if ( output_config == AUDIO_CONFIG_FOA || output_config == AUDIO_CONFIG_HOA2 || output_config == AUDIO_CONFIG_HOA3 ) +#endif + if ( st_ivas->ism_mode == ISM_MODE_PARAM ) { - *renderer_type = RENDERER_SBA_LINEAR_ENC; + *renderer_type = RENDERER_PARAM_ISM; + if ( output_config == AUDIO_CONFIG_MONO ) + { + *renderer_type = RENDERER_MONO_DOWNMIX; + } + else if ( output_config == AUDIO_CONFIG_STEREO ) + { + *renderer_type = RENDERER_DISABLE; + } + else if ( output_config == AUDIO_CONFIG_FOA || output_config == AUDIO_CONFIG_HOA2 || output_config == AUDIO_CONFIG_HOA3 ) + { + *renderer_type = RENDERER_SBA_LINEAR_ENC; + *internal_config = AUDIO_CONFIG_7_1_4; + } } - else if ( output_config == AUDIO_CONFIG_EXTERNAL ) + else /* ISM_MODE_DISC */ { - *renderer_type = RENDERER_DISABLE; + *renderer_type = RENDERER_TD_PANNING; + if ( output_config == AUDIO_CONFIG_MONO ) + { + *renderer_type = RENDERER_MONO_DOWNMIX; + } + else if ( output_config == AUDIO_CONFIG_FOA || output_config == AUDIO_CONFIG_HOA2 || output_config == AUDIO_CONFIG_HOA3 ) + { + *renderer_type = RENDERER_SBA_LINEAR_ENC; + } + else if ( output_config == AUDIO_CONFIG_EXTERNAL ) + { + *renderer_type = RENDERER_DISABLE; + } } +#ifdef NON_DIEGETIC_PAN } +#endif } else if ( st_ivas->ivas_format == MASA_FORMAT || st_ivas->ivas_format == SBA_FORMAT ) { diff --git a/lib_dec/ivas_stat_dec.h b/lib_dec/ivas_stat_dec.h index 0e2f01c574c0189fe64d0fb355cb1b705424ea30..3a0259c1c3bad85a6c0344822b88426f409d77b7 100755 --- a/lib_dec/ivas_stat_dec.h +++ b/lib_dec/ivas_stat_dec.h @@ -1175,7 +1175,12 @@ typedef struct decoder_config_structure int16_t Opt_Headrotation; /* indicates whether head-rotation is used */ int16_t Opt_RendConfigCustom; /* indicates whether Renderer configuration custom setup is used */ int16_t orientation_tracking; /* indicates orientation tracking type */ +#ifdef NON_DIEGETIC_PAN + int16_t Opt_non_diegetic_pan; /* indicates diegetic or not */ + float non_diegetic_pan_gain; /* non diegetic panning gain*/ +#else float no_diegetic_pan; +#endif int16_t Opt_AMR_WB; /* flag indicating AMR-WB IO mode */ /* temp. development parameters */ diff --git a/lib_dec/lib_dec.c b/lib_dec/lib_dec.c index ee315bf399d5b22f8ed7716a4d41e6fbe988cdec..2209e118deece567ce23a22bb1cc9f4575a529c6 100644 --- a/lib_dec/lib_dec.c +++ b/lib_dec/lib_dec.c @@ -95,7 +95,11 @@ static void store_JbmData( IVAS_DEC_VOIP *hVoIP, JB4_DATAUNIT_HANDLE dataUnit, c #endif static ivas_error evs_dec_main( Decoder_Struct *st_ivas, const int16_t nOutSamples, int16_t *pcmBuf ); static ivas_error input_format_API_to_internal( IVAS_DEC_INPUT_FORMAT input_format, int16_t *bitstream_format_internal, int16_t *sdp_hf_only, const bool is_voip_enabled ); -static void init_decoder_config( DECODER_CONFIG_HANDLE hDecoderConfig, const int16_t orientation_tracking, const float no_diegetic_pan ); +#ifdef NON_DIEGETIC_PAN +static void init_decoder_config( DECODER_CONFIG_HANDLE hDecoderConfig, const int16_t orientation_tracking ); +#else +static void init_decoder_config( DECODER_CONFIG_HANDLE hDecoderConfig, const int16_t orientation_tracking, const float non_diegetic_pan ); +#endif /*---------------------------------------------------------------------* @@ -106,10 +110,14 @@ static void init_decoder_config( DECODER_CONFIG_HANDLE hDecoderConfig, const int /* may return an error but may still have allocated memory - thus run Close also in case of error to release memory */ ivas_error IVAS_DEC_Open( - IVAS_DEC_HANDLE *phIvasDec, /* i/o: pointer to an IVAS decoder handle to be opened */ - const IVAS_DEC_MODE mode, /* i : compatibility mode (EVS or IVAS) */ - const int16_t orientation_tracking, /* i : orientation tracking type */ - float no_diegetic_pan ) + IVAS_DEC_HANDLE *phIvasDec, /* i/o: pointer to an IVAS decoder handle to be opened */ + const IVAS_DEC_MODE mode, /* i : compatibility mode (EVS or IVAS) */ + const int16_t orientation_tracking /* i : orientation tracking type */ +#ifndef NON_DIEGETIC_PAN + , + float no_diegetic_pan /* i : non diegetic panning gain */ +#endif +) { IVAS_DEC_HANDLE hIvasDec; Decoder_Struct *st_ivas; @@ -162,7 +170,11 @@ ivas_error IVAS_DEC_Open( st_ivas = hIvasDec->st_ivas; /* initialize Decoder Config. handle */ +#ifdef NON_DIEGETIC_PAN + init_decoder_config( hIvasDec->st_ivas->hDecoderConfig, orientation_tracking ); +#else init_decoder_config( hIvasDec->st_ivas->hDecoderConfig, orientation_tracking, no_diegetic_pan ); +#endif /* initialize pointers to handles to NULL */ ivas_initialize_handles_dec( st_ivas ); @@ -214,8 +226,12 @@ ivas_error IVAS_DEC_Open( static void init_decoder_config( DECODER_CONFIG_HANDLE hDecoderConfig, /* i/o: configuration structure */ - const int16_t orientation_tracking, - const float no_diegetic_pan ) + const int16_t orientation_tracking +#ifndef NON_DIEGETIC_PAN + , + const float no_diegetic_pan +#endif +) { hDecoderConfig->Opt_AMR_WB = 0; hDecoderConfig->nchan_out = 1; @@ -225,7 +241,12 @@ static void init_decoder_config( hDecoderConfig->Opt_Headrotation = 0; hDecoderConfig->Opt_RendConfigCustom = 0; hDecoderConfig->orientation_tracking = orientation_tracking; +#ifdef NON_DIEGETIC_PAN + hDecoderConfig->Opt_non_diegetic_pan = 0; + hDecoderConfig->non_diegetic_pan_gain = 0; +#else hDecoderConfig->no_diegetic_pan = no_diegetic_pan; +#endif return; } @@ -388,7 +409,13 @@ ivas_error IVAS_DEC_Configure( const int16_t customLsOutputEnabled, /* i : enable custom loudspeaker setup handle */ const int16_t hrtfReaderEnabled, /* i : enable HRTF binary file input */ const int16_t enableHeadRotation, /* i : enable head rotation for binaural output */ - const int16_t renderConfigEnabled /* i : enable Renderer config. file for binaural output */ +#ifdef NON_DIEGETIC_PAN + const int16_t renderConfigEnabled, /* i : enable Renderer config. file for binaural output */ + const int16_t Opt_non_diegetic_pan, /* i : diegetic or not */ + const float non_diegetic_pan_gain /* i : non diegetic panning gain */ +#else + const int16_t renderConfigEnabled /* i : enable Renderer config. file for binaural output */ +#endif ) { Decoder_Struct *st_ivas; @@ -406,10 +433,18 @@ ivas_error IVAS_DEC_Configure( return IVAS_ERR_WRONG_PARAMS; } +#ifdef NON_DIEGETIC_PAN + if ( hIvasDec->mode == IVAS_DEC_MODE_EVS && !( ( outputFormat == IVAS_DEC_OUTPUT_MONO && Opt_non_diegetic_pan == 0 ) || + ( outputFormat == IVAS_DEC_OUTPUT_STEREO && Opt_non_diegetic_pan == 1 ) ) ) + { + return IVAS_ERR_WRONG_MODE; + } +#else if ( hIvasDec->mode == IVAS_DEC_MODE_EVS && outputFormat != IVAS_DEC_OUTPUT_MONO ) { return IVAS_ERR_WRONG_MODE; } +#endif st_ivas = hIvasDec->st_ivas; @@ -438,6 +473,10 @@ ivas_error IVAS_DEC_Configure( hDecoderConfig->Opt_Headrotation = enableHeadRotation; hDecoderConfig->Opt_HRTF_binary = hrtfReaderEnabled; hDecoderConfig->Opt_RendConfigCustom = renderConfigEnabled; +#ifdef NON_DIEGETIC_PAN + hDecoderConfig->Opt_non_diegetic_pan = Opt_non_diegetic_pan; + hDecoderConfig->non_diegetic_pan_gain = non_diegetic_pan_gain; +#endif /* Set decoder parameters to initial values */ if ( ( error = ivas_init_decoder_front( st_ivas ) ) != IVAS_ERR_OK ) @@ -830,6 +869,9 @@ ivas_error IVAS_DEC_GetObjectMetadata( metadata->gainFactor = 1.f; metadata->yaw = 0.f; metadata->pitch = 0.f; +#ifdef ISM_NON_DIEGETIC_PAN + metadata->non_diegetic_flag = 0; +#endif } else { @@ -840,6 +882,9 @@ ivas_error IVAS_DEC_GetObjectMetadata( metadata->pitch = hIsmMeta->pitch; metadata->spread = 0.f; metadata->gainFactor = 1.f; +#ifdef ISM_NON_DIEGETIC_PAN + metadata->non_diegetic_flag = hIsmMeta->non_diegetic_flag; +#endif } return IVAS_ERR_OK; @@ -1928,7 +1973,17 @@ static ivas_error printConfigInfo_dec( if ( st_ivas->ivas_format == MONO_FORMAT ) { - fprintf( stdout, "Output configuration: mono EVS bit-exact decoding\n" ); +#ifdef NON_DIEGETIC_PAN + if ( st_ivas->hDecoderConfig->Opt_non_diegetic_pan ) + { + fprintf( stdout, "Output configuration: mono EVS bit-exact decoding to stereo\n" ); + fprintf( stdout, "Non-diegetic panning: %.2f\n", st_ivas->hDecoderConfig->non_diegetic_pan_gain * 90.f ); + } + else +#endif + { + fprintf( stdout, "Output configuration: mono EVS bit-exact decoding\n" ); + } } else { @@ -1967,6 +2022,7 @@ static ivas_error printConfigInfo_dec( fprintf( stdout, "Input configuration: %s\n", config_str ); } } + get_channel_config( st_ivas->hDecoderConfig->output_config, &config_str[0] ); fprintf( stdout, "Output configuration: %s\n", config_str ); @@ -2003,6 +2059,13 @@ static ivas_error printConfigInfo_dec( break; } } + +#ifdef NON_DIEGETIC_PAN + if ( st_ivas->hDecoderConfig->Opt_non_diegetic_pan ) + { + fprintf( stdout, "Non-diegetic panning: %.2f\n", st_ivas->hDecoderConfig->non_diegetic_pan_gain * 90.f ); + } +#endif } return IVAS_ERR_OK; @@ -2106,7 +2169,12 @@ static ivas_error evs_dec_main( int16_t *pcmBuf ) { DEC_CORE_HANDLE *hCoreCoder; +#ifdef NON_DIEGETIC_PAN + float output[MAX_OUTPUT_CHANNELS_IN_DIEGETIC_PAN][L_FRAME48k]; + float mixer_left, mixer_rigth; +#else float output[L_FRAME48k]; +#endif ivas_error error; error = IVAS_ERR_OK; @@ -2121,14 +2189,22 @@ static ivas_error evs_dec_main( { if ( hCoreCoder[0]->Opt_AMR_WB ) { +#ifdef NON_DIEGETIC_PAN + if ( ( error = amr_wb_dec( hCoreCoder[0], st_ivas->mem_hp20_out[0], output[0] ) ) != IVAS_ERR_OK ) +#else if ( ( error = amr_wb_dec( hCoreCoder[0], st_ivas->mem_hp20_out[0], output ) ) != IVAS_ERR_OK ) +#endif { return error; } } else { +#ifdef NON_DIEGETIC_PAN + if ( ( error = evs_dec( hCoreCoder[0], st_ivas->mem_hp20_out[0], output[0], FRAMEMODE_NORMAL ) ) != IVAS_ERR_OK ) +#else if ( ( error = evs_dec( hCoreCoder[0], st_ivas->mem_hp20_out[0], output, FRAMEMODE_NORMAL ) ) != IVAS_ERR_OK ) +#endif { return error; } @@ -2138,21 +2214,33 @@ static ivas_error evs_dec_main( { if ( hCoreCoder[0]->bfi == 0 ) { +#ifdef NON_DIEGETIC_PAN + if ( ( error = evs_dec( hCoreCoder[0], st_ivas->mem_hp20_out[0], output[0], FRAMEMODE_NORMAL ) ) != IVAS_ERR_OK ) +#else if ( ( error = evs_dec( hCoreCoder[0], st_ivas->mem_hp20_out[0], output, FRAMEMODE_NORMAL ) ) != IVAS_ERR_OK ) +#endif { return error; } } else if ( hCoreCoder[0]->bfi == 2 ) { +#ifdef NON_DIEGETIC_PAN + if ( ( error = evs_dec( hCoreCoder[0], st_ivas->mem_hp20_out[0], output[0], FRAMEMODE_FUTURE ) ) != IVAS_ERR_OK ) +#else if ( ( error = evs_dec( hCoreCoder[0], st_ivas->mem_hp20_out[0], output, FRAMEMODE_FUTURE ) ) != IVAS_ERR_OK ) +#endif { return error; } } else { +#ifdef NON_DIEGETIC_PAN + if ( ( error = evs_dec( hCoreCoder[0], st_ivas->mem_hp20_out[0], output[0], FRAMEMODE_MISSING ) ) != IVAS_ERR_OK ) +#else if ( ( error = evs_dec( hCoreCoder[0], st_ivas->mem_hp20_out[0], output, FRAMEMODE_MISSING ) ) != IVAS_ERR_OK ) +#endif { return error; } @@ -2161,11 +2249,25 @@ static ivas_error evs_dec_main( st_ivas->BER_detect = hCoreCoder[0]->BER_detect; +#ifdef NON_DIEGETIC_PAN + if ( st_ivas->renderer_type == RENDERER_NON_DIEGETIC_DOWNMIX ) + { + mixer_left = ( st_ivas->hDecoderConfig->non_diegetic_pan_gain + 1.f ) * 0.5f; + mixer_rigth = 1.f - mixer_left; + v_multc( output[0], mixer_rigth, output[1], nOutSamples ); + v_multc( output[0], mixer_left, output[0], nOutSamples ); + } +#endif + /* convert 'float' output data to 'short' */ #ifdef DEBUGGING st_ivas->noClipping += #endif - syn_output( output, nOutSamples, pcmBuf ); +#ifdef NON_DIEGETIC_PAN + ivas_syn_output( output, nOutSamples, st_ivas->hDecoderConfig->nchan_out, pcmBuf ); +#else + syn_output( output, nOutSamples, pcmBuf ); +#endif return error; } diff --git a/lib_dec/lib_dec.h b/lib_dec/lib_dec.h index f220184ca15496960251a936468eb5c6eb960ce2..06fc8820ae8537027eb77e76bd98ecc71fdc7fcb 100644 --- a/lib_dec/lib_dec.h +++ b/lib_dec/lib_dec.h @@ -127,8 +127,11 @@ typedef ivas_error ( *JbmTraceFileWriterFn )( const void *data, void *writer ); ivas_error IVAS_DEC_Open( IVAS_DEC_HANDLE *phIvasDec, /* i/o: pointer to an IVAS decoder handle to be opened */ IVAS_DEC_MODE mode, /* i : compatibility mode (EVS or IVAS) */ - const int16_t orientation_tracking, /* i : orientation tracking type */ - float no_diegetic_pan + const int16_t orientation_tracking /* i : orientation tracking type */ +#ifndef NON_DIEGETIC_PAN + , + float no_diegetic_pan_gain /* i : non diegetic panning gain */ +#endif ); /*! r: error code */ @@ -139,7 +142,13 @@ ivas_error IVAS_DEC_Configure( const int16_t customLsOutputEnabled, /* i : enable custom loudspeaker setup handle */ const int16_t hrtfReaderEnabled, /* i : enable HRTF binary file input */ const int16_t enableHeadRotation, /* i : enable head rotation for binaural output */ +#ifdef NON_DIEGETIC_PAN + const int16_t renderConfigEnabled, /* i : enable Renderer config. file for binaural output */ + const int16_t Opt_non_diegetic_pan, /* i : diegetic or not */ + const float non_diegetic_pan_gain /* i : non diegetic panning gain */ +#else const int16_t renderConfigEnabled /* i : enable Renderer config. file for binaural output */ +#endif ); void IVAS_DEC_Close( diff --git a/lib_enc/ivas_ism_enc.c b/lib_enc/ivas_ism_enc.c index 4a56252be876fa73e3a967c469b87caf11f2bd36..5c310ac30fcb3a66be6e666712db0a70fd870d67 100644 --- a/lib_enc/ivas_ism_enc.c +++ b/lib_enc/ivas_ism_enc.c @@ -402,7 +402,11 @@ ivas_error ivas_ism_enc_config( st_ivas->nSCE = st_ivas->nchan_transport; st_ivas->nCPE = 0; +#ifdef ISM_NON_DIEGETIC_PAN + if ( ( error = ivas_ism_config( st_ivas->hEncoderConfig->ivas_total_brate, st_ivas->nchan_transport, st_ivas->hEncoderConfig->nchan_inp, NULL, 0, NULL, NULL, element_brate_tmp, NULL, NULL ) ) != IVAS_ERR_OK ) +#else if ( ( error = ivas_ism_config( st_ivas->hEncoderConfig->ivas_total_brate, st_ivas->nchan_transport, st_ivas->hEncoderConfig->nchan_inp, NULL, NULL, NULL, element_brate_tmp, NULL, NULL ) ) != IVAS_ERR_OK ) +#endif { return error; } diff --git a/lib_enc/ivas_ism_metadata_enc.c b/lib_enc/ivas_ism_metadata_enc.c index a4678d6bbb1dcba739f6876f2e81f806d778c94b..732ffec21b7a2cf68d889caec204e9a9da3b0811 100644 --- a/lib_enc/ivas_ism_metadata_enc.c +++ b/lib_enc/ivas_ism_metadata_enc.c @@ -88,7 +88,12 @@ ivas_error ivas_set_ism_metadata( const float elevation, /* i : elevation */ const float radius_meta, /* i : radius */ const float yaw, /* i : yaw */ - const float pitch /* i : pitch */ +#ifdef ISM_NON_DIEGETIC_PAN + const float pitch, /* i : pitch */ + const int16_t non_diegetic_flag /* i : non-diegetic object flag*/ +#else + const float pitch /* i : pitch */ +#endif ) { if ( hIsmMeta == NULL ) @@ -104,6 +109,9 @@ ivas_error ivas_set_ism_metadata( hIsmMeta->radius = radius_meta; hIsmMeta->yaw = yaw; hIsmMeta->pitch = pitch; +#ifdef ISM_NON_DIEGETIC_PAN + hIsmMeta->non_diegetic_flag = non_diegetic_flag; +#endif return IVAS_ERR_OK; } @@ -194,6 +202,9 @@ ivas_error ivas_ism_metadata_enc( ISM_METADATA_HANDLE hIsmMetaData; int32_t element_brate[MAX_NUM_OBJECTS], total_brate[MAX_NUM_OBJECTS]; int16_t ism_metadata_flag_global; +#ifdef ISM_NON_DIEGETIC_PAN + int16_t non_diegetic_flag_global; +#endif int16_t ism_imp[MAX_NUM_OBJECTS]; int16_t nbands, nblocks; ivas_error error; @@ -204,6 +215,9 @@ ivas_error ivas_ism_metadata_enc( /* initialization */ ism_metadata_flag_global = 0; +#ifdef ISM_NON_DIEGETIC_PAN + non_diegetic_flag_global = 0; +#endif set_s( nb_bits_metadata, 0, nchan_transport ); set_s( flag_abs_azimuth, 0, nchan_ism ); set_s( flag_abs_elevation, 0, nchan_ism ); @@ -289,10 +303,26 @@ ivas_error ivas_ism_metadata_enc( } push_indice( hBstr, IND_ISM_NUM_OBJECTS, 0, 1 ); +#ifdef ISM_NON_DIEGETIC_PAN + for ( ch = 0; ch < nchan_ism; ch++ ) + { + ism_metadata_flag_global |= hIsmMeta[ch]->ism_metadata_flag; + non_diegetic_flag_global |= hIsmMeta[ch]->non_diegetic_flag; + } +#endif + /* write extended metadata presence flag */ if ( ism_total_brate >= ISM_EXTENDED_METADATA_BRATE ) { push_indice( hBstr, IND_ISM_EXTENDED_FLAG, ism_extended_metadata_flag, ISM_EXTENDED_METADATA_BITS ); + +#ifdef ISM_NON_DIEGETIC_PAN + /* Write global non-diegetic object flag */ + if ( ism_extended_metadata_flag ) + { + push_indice( hBstr, IND_ISM_EXTENDED_NDP_FLAG, non_diegetic_flag_global, ISM_EXTENDED_METADATA_BITS ); + } +#endif } /* write ISM metadata flag (one per object) */ @@ -301,11 +331,12 @@ ivas_error ivas_ism_metadata_enc( push_indice( hBstr, IND_ISM_METADATA_FLAG, ism_imp[ch], ISM_METADATA_FLAG_BITS ); } +#ifndef ISM_NON_DIEGETIC_PAN for ( ch = 0; ch < nchan_ism; ch++ ) { ism_metadata_flag_global |= hIsmMeta[ch]->ism_metadata_flag; } - +#endif if ( ism_mode == ISM_MODE_DISC ) { /* write VAD flag */ @@ -347,32 +378,61 @@ ivas_error ivas_ism_metadata_enc( * Quantize and encode azimuth and elevation *----------------------------------------------------------------*/ - if ( ism_mode == ISM_MODE_DISC ) +#ifdef ISM_NON_DIEGETIC_PAN + if ( ism_extended_metadata_flag && non_diegetic_flag_global ) { - idx_angle1_abs = ism_quant_meta( hIsmMetaData->azimuth, &valQ, ism_azimuth_borders, ISM_Q_STEP, ISM_Q_STEP_BORDER, 1 << ISM_AZIMUTH_NBITS ); - idx_angle2_abs = ism_quant_meta( hIsmMetaData->elevation, &valQ, ism_elevation_borders, ISM_Q_STEP, ISM_Q_STEP_BORDER, 1 << ISM_ELEVATION_NBITS ); + /* Write non-diegetic flag for each object */ + push_indice( hBstr, IND_ISM_NDP_FLAG, hIsmMeta[ch]->non_diegetic_flag, ISM_METADATA_IS_NDP_BITS ); } - else /* ISM_MODE_PARAM */ + + if ( hIsmMeta[ch]->non_diegetic_flag ) { - idx_angle1_abs = hParamIsm->azi_index[ch]; - idx_angle2_abs = hParamIsm->ele_index[ch]; + /* Map azimuth to panning range [-90:90] */ + if ( hIsmMetaData->azimuth > 90.0f ) + { + hIsmMetaData->azimuth = 180.0f - hIsmMetaData->azimuth; + } + + if ( hIsmMetaData->azimuth < -90.0f ) + { + hIsmMetaData->azimuth = -180.0f - hIsmMetaData->azimuth; + } + + idx_angle1_abs = ism_quant_meta( hIsmMetaData->azimuth, &valQ, ism_azimuth_borders, ISM_Q_STEP, ISM_Q_STEP_BORDER, 1 << ISM_AZIMUTH_NBITS ); + encode_angle_indices( hBstr, &( hIsmMetaData->position_angle ), hIsmMetaData->last_ism_metadata_flag, hSCE[0]->hCoreCoder[0]->ini_frame, idx_angle1_abs, 0, &flag_abs_azimuth[ch], NULL ); } + else + { +#endif + if ( ism_mode == ISM_MODE_DISC ) + { + idx_angle1_abs = ism_quant_meta( hIsmMetaData->azimuth, &valQ, ism_azimuth_borders, ISM_Q_STEP, ISM_Q_STEP_BORDER, 1 << ISM_AZIMUTH_NBITS ); + idx_angle2_abs = ism_quant_meta( hIsmMetaData->elevation, &valQ, ism_elevation_borders, ISM_Q_STEP, ISM_Q_STEP_BORDER, 1 << ISM_ELEVATION_NBITS ); + } + else /* ISM_MODE_PARAM */ + { + idx_angle1_abs = hParamIsm->azi_index[ch]; + idx_angle2_abs = hParamIsm->ele_index[ch]; + } - encode_angle_indices( hBstr, &( hIsmMetaData->position_angle ), hIsmMetaData->last_ism_metadata_flag, hSCE[0]->hCoreCoder[0]->ini_frame, idx_angle1_abs, idx_angle2_abs, &flag_abs_azimuth[ch], &flag_abs_elevation[ch] ); + encode_angle_indices( hBstr, &( hIsmMetaData->position_angle ), hIsmMetaData->last_ism_metadata_flag, hSCE[0]->hCoreCoder[0]->ini_frame, idx_angle1_abs, idx_angle2_abs, &flag_abs_azimuth[ch], &flag_abs_elevation[ch] ); - /*----------------------------------------------------------------* - * Quantize and encode radius, yaw, and pitch - *----------------------------------------------------------------*/ + /*----------------------------------------------------------------* + * Quantize and encode radius, yaw, and pitch + *----------------------------------------------------------------*/ - if ( ism_mode == ISM_MODE_DISC && ism_extended_metadata_flag ) - { - idx_angle1_abs = ism_quant_meta( hIsmMetaData->yaw, &valQ, ism_azimuth_borders, ISM_Q_STEP, ISM_Q_STEP_BORDER, 1 << ISM_AZIMUTH_NBITS ); - idx_angle2_abs = ism_quant_meta( hIsmMetaData->pitch, &valQ, ism_elevation_borders, ISM_Q_STEP, ISM_Q_STEP_BORDER, 1 << ISM_ELEVATION_NBITS ); - idx_radius_abs = usquant( hIsmMetaData->radius, &valQ, ISM_RADIUS_MIN, ISM_RADIUS_DELTA, 1 << ISM_RADIUS_NBITS ); + if ( ism_mode == ISM_MODE_DISC && ism_extended_metadata_flag ) + { + idx_angle1_abs = ism_quant_meta( hIsmMetaData->yaw, &valQ, ism_azimuth_borders, ISM_Q_STEP, ISM_Q_STEP_BORDER, 1 << ISM_AZIMUTH_NBITS ); + idx_angle2_abs = ism_quant_meta( hIsmMetaData->pitch, &valQ, ism_elevation_borders, ISM_Q_STEP, ISM_Q_STEP_BORDER, 1 << ISM_ELEVATION_NBITS ); + idx_radius_abs = usquant( hIsmMetaData->radius, &valQ, ISM_RADIUS_MIN, ISM_RADIUS_DELTA, 1 << ISM_RADIUS_NBITS ); - encode_angle_indices( hBstr, &( hIsmMetaData->orientation_angle ), hIsmMetaData->last_ism_metadata_flag, hSCE[0]->hCoreCoder[0]->ini_frame, idx_angle1_abs, idx_angle2_abs, &flag_abs_yaw[ch], &flag_abs_pitch[ch] ); - encode_radius( hBstr, &hIsmMetaData->last_radius_idx, &hIsmMetaData->radius_diff_cnt, hIsmMetaData->last_ism_metadata_flag, idx_radius_abs, &flag_abs_radius[ch] ); + encode_angle_indices( hBstr, &( hIsmMetaData->orientation_angle ), hIsmMetaData->last_ism_metadata_flag, hSCE[0]->hCoreCoder[0]->ini_frame, idx_angle1_abs, idx_angle2_abs, &flag_abs_yaw[ch], &flag_abs_pitch[ch] ); + encode_radius( hBstr, &hIsmMetaData->last_radius_idx, &hIsmMetaData->radius_diff_cnt, hIsmMetaData->last_ism_metadata_flag, idx_radius_abs, &flag_abs_radius[ch] ); + } +#ifdef ISM_NON_DIEGETIC_PAN } +#endif /* save number of metadata bits written */ if ( ism_mode == ISM_MODE_DISC ) @@ -516,7 +576,11 @@ ivas_error ivas_ism_metadata_enc( * Configuration and decision about bitrates per channel *----------------------------------------------------------------*/ +#ifdef ISM_NON_DIEGETIC_PAN + if ( ( error = ivas_ism_config( ism_total_brate, nchan_transport, nchan_ism, hIsmMeta, ism_extended_metadata_flag, localVAD, ism_imp, element_brate, total_brate, nb_bits_metadata ) ) != IVAS_ERR_OK ) +#else if ( ( error = ivas_ism_config( ism_total_brate, nchan_transport, nchan_ism, hIsmMeta, localVAD, ism_imp, element_brate, total_brate, nb_bits_metadata ) ) != IVAS_ERR_OK ) +#endif { return error; } @@ -634,7 +698,11 @@ ivas_error ivas_ism_metadata_enc_create( #endif } +#ifdef ISM_NON_DIEGETIC_PAN + if ( ( error = ivas_ism_config( st_ivas->hEncoderConfig->ivas_total_brate, nchan_transport, n_ISms, NULL, 0, NULL, NULL, element_brate_tmp, NULL, NULL ) ) != IVAS_ERR_OK ) +#else if ( ( error = ivas_ism_config( st_ivas->hEncoderConfig->ivas_total_brate, nchan_transport, n_ISms, NULL, NULL, NULL, element_brate_tmp, NULL, NULL ) ) != IVAS_ERR_OK ) +#endif { return error; } @@ -870,116 +938,123 @@ static void encode_angle_indices( * Elevation/pitch index encoding *----------------------------------------------------------------*/ - idx_angle2 = idx_angle2_abs; - nbits_diff_angle2 = 0; - *flag_abs_angle2 = 0; /* differential coding by default */ - if ( angle->angle2_diff_cnt == ISM_FEC_MAX /* make differential encoding in ISM_FEC_MAX consecutive frames at maximum (in order to control the decoding in FEC) */ - || last_ism_metadata_flag == 0 /* If last frame had no metadata coded, do not use differential coding */ - ) +#ifdef ISM_NON_DIEGETIC_PAN + if ( flag_abs_angle2 ) { - *flag_abs_angle2 = 1; - } - - /* note: elevation/pitch is coded starting from the second frame only (it is meaningless in the init_frame) */ - if ( ini_frame == 0 ) - { - *flag_abs_angle2 = 1; - angle->last_angle2_idx = idx_angle2_abs; - } - - diff = idx_angle2_abs - angle->last_angle2_idx; - - /* avoid absolute coding of elevation/pitch if absolute coding was already used for azimuth/yaw */ - if ( *flag_abs_angle1 == 1 ) - { - int16_t diff_orig = diff; - - *flag_abs_angle2 = 0; - - - if ( diff >= 0 ) - { - diff = min( diff, ISM_MAX_ELEVATION_DIFF_IDX ); - } - else +#endif + idx_angle2 = idx_angle2_abs; + nbits_diff_angle2 = 0; + *flag_abs_angle2 = 0; /* differential coding by default */ + if ( angle->angle2_diff_cnt == ISM_FEC_MAX /* make differential encoding in ISM_FEC_MAX consecutive frames at maximum (in order to control the decoding in FEC) */ + || last_ism_metadata_flag == 0 /* If last frame had no metadata coded, do not use differential coding */ + ) { - diff = -1 * min( -diff, ISM_MAX_ELEVATION_DIFF_IDX ); + *flag_abs_angle2 = 1; } - if ( last_ism_metadata_flag == 0 || abs( diff_orig - diff ) > ISM_MAX_ELEVATION_DIFF_IDX ) + /* note: elevation/pitch is coded starting from the second frame only (it is meaningless in the init_frame) */ + if ( ini_frame == 0 ) { - angle->angle2_diff_cnt = ISM_FEC_MAX - 1; + *flag_abs_angle2 = 1; + angle->last_angle2_idx = idx_angle2_abs; } - } - /* try differential coding */ - if ( *flag_abs_angle2 == 0 ) - { - if ( diff == 0 ) - { - idx_angle2 = 0; - nbits_diff_angle2 = 1; - } - else if ( ABSVAL( diff ) <= ISM_MAX_ELEVATION_DIFF_IDX ) + diff = idx_angle2_abs - angle->last_angle2_idx; + + /* avoid absolute coding of elevation/pitch if absolute coding was already used for azimuth/yaw */ + if ( *flag_abs_angle1 == 1 ) { - idx_angle2 = 1 << 1; - nbits_diff_angle2 = 1; + int16_t diff_orig = diff; - if ( diff < 0 ) + *flag_abs_angle2 = 0; + + + if ( diff >= 0 ) { - idx_angle2 += 1; /* negative sign */ - diff *= -1; + diff = min( diff, ISM_MAX_ELEVATION_DIFF_IDX ); } else { - idx_angle2 += 0; /* positive sign */ + diff = -1 * min( -diff, ISM_MAX_ELEVATION_DIFF_IDX ); } - idx_angle2 = idx_angle2 << diff; - nbits_diff_angle2++; - - /* unary coding of "diff */ - idx_angle2 += ( ( 1 << diff ) - 1 ); - nbits_diff_angle2 += diff; + if ( last_ism_metadata_flag == 0 || abs( diff_orig - diff ) > ISM_MAX_ELEVATION_DIFF_IDX ) + { + angle->angle2_diff_cnt = ISM_FEC_MAX - 1; + } + } - if ( nbits_diff_angle2 < ISM_ELEVATION_NBITS ) + /* try differential coding */ + if ( *flag_abs_angle2 == 0 ) + { + if ( diff == 0 ) { - /* add stop bit */ - idx_angle2 = idx_angle2 << 1; + idx_angle2 = 0; + nbits_diff_angle2 = 1; + } + else if ( ABSVAL( diff ) <= ISM_MAX_ELEVATION_DIFF_IDX ) + { + idx_angle2 = 1 << 1; + nbits_diff_angle2 = 1; + + if ( diff < 0 ) + { + idx_angle2 += 1; /* negative sign */ + diff *= -1; + } + else + { + idx_angle2 += 0; /* positive sign */ + } + + idx_angle2 = idx_angle2 << diff; nbits_diff_angle2++; + + /* unary coding of "diff */ + idx_angle2 += ( ( 1 << diff ) - 1 ); + nbits_diff_angle2 += diff; + + if ( nbits_diff_angle2 < ISM_ELEVATION_NBITS ) + { + /* add stop bit */ + idx_angle2 = idx_angle2 << 1; + nbits_diff_angle2++; + } } + else + { + *flag_abs_angle2 = 1; + } + } + + /* update counter */ + if ( *flag_abs_angle2 == 0 ) + { + angle->angle2_diff_cnt++; + angle->angle2_diff_cnt = min( angle->angle2_diff_cnt, ISM_FEC_MAX ); } else { - *flag_abs_angle2 = 1; + angle->angle2_diff_cnt = 0; } - } - - /* update counter */ - if ( *flag_abs_angle2 == 0 ) - { - angle->angle2_diff_cnt++; - angle->angle2_diff_cnt = min( angle->angle2_diff_cnt, ISM_FEC_MAX ); - } - else - { - angle->angle2_diff_cnt = 0; - } - /* Write elevation */ - if ( *flag_abs_angle1 == 0 ) /* do not write "flag_abs_elevation/pitch" if "flag_abs_azimuth/yaw == 1" */ - { - push_indice( hBstr, IND_ISM_ELEVATION_DIFF_FLAG, *flag_abs_angle2, 1 ); - } + /* Write elevation */ + if ( *flag_abs_angle1 == 0 ) /* do not write "flag_abs_elevation/pitch" if "flag_abs_azimuth/yaw == 1" */ + { + push_indice( hBstr, IND_ISM_ELEVATION_DIFF_FLAG, *flag_abs_angle2, 1 ); + } - if ( *flag_abs_angle2 ) - { - push_indice( hBstr, IND_ISM_ELEVATION, idx_angle2, ISM_ELEVATION_NBITS ); - } - else - { - push_indice( hBstr, IND_ISM_ELEVATION, idx_angle2, nbits_diff_angle2 ); + if ( *flag_abs_angle2 ) + { + push_indice( hBstr, IND_ISM_ELEVATION, idx_angle2, ISM_ELEVATION_NBITS ); + } + else + { + push_indice( hBstr, IND_ISM_ELEVATION, idx_angle2, nbits_diff_angle2 ); + } +#ifdef ISM_NON_DIEGETIC_PAN } +#endif /*----------------------------------------------------------------* * Updates diff --git a/lib_enc/lib_enc.c b/lib_enc/lib_enc.c old mode 100644 new mode 100755 index 45d4c856837432451c385ce5b194b79591cb519e..c0c23b8dd9cf7a725e55b22a37dd78d140bcc8a7 --- a/lib_enc/lib_enc.c +++ b/lib_enc/lib_enc.c @@ -430,7 +430,11 @@ ivas_error IVAS_ENC_FeedObjectMetadata( return IVAS_ERR_INDEX_OUT_OF_BOUNDS; } +#ifdef ISM_NON_DIEGETIC_PAN + error = ivas_set_ism_metadata( hIvasEnc->st_ivas->hIsmMetaData[ismIndex], metadata.azimuth, metadata.elevation, metadata.radius, metadata.yaw, metadata.pitch, metadata.non_diegetic_flag ); +#else error = ivas_set_ism_metadata( hIvasEnc->st_ivas->hIsmMetaData[ismIndex], metadata.azimuth, metadata.elevation, metadata.radius, metadata.yaw, metadata.pitch ); +#endif if ( error != IVAS_ERR_OK ) { diff --git a/lib_rend/ivas_objectRenderer.c b/lib_rend/ivas_objectRenderer.c index 0618534448deec903f1f016e8d2f8030f84e028f..540a03f4518a7b6d7a5b118c39ed7879a3578762 100644 --- a/lib_rend/ivas_objectRenderer.c +++ b/lib_rend/ivas_objectRenderer.c @@ -34,6 +34,9 @@ #include #include "options.h" #include "prot.h" +#ifdef ISM_NON_DIEGETIC_PAN +#include "ivas_prot.h" +#endif #include "ivas_prot_rend.h" #include #include "ivas_rom_com.h" @@ -48,8 +51,10 @@ *---------------------------------------------------------------------*/ static void TDREND_Clear_Update_flags( BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd ); + static void angles_to_vec( const float radius, const float azimuth, const float elevation, float *vec ); + /*---------------------------------------------------------------------* * ivas_td_binaural_open_unwrap() * @@ -327,6 +332,10 @@ ivas_error TDREND_GetMix( float hrf_left_delta[SFX_SPAT_BIN_MAX_FILTER_LENGTH]; float hrf_right_delta[SFX_SPAT_BIN_MAX_FILTER_LENGTH]; int16_t intp_count; +#ifdef ISM_NON_DIEGETIC_PAN + float pan_left, pan_right; +#endif + error = IVAS_ERR_OK; /* Clear the output buffer to accumulate rendered sources */ @@ -347,7 +356,11 @@ ivas_error TDREND_GetMix( SrcRend_p = Src_p->SrcRend_p; /* Update rendering params if needed */ +#ifdef ISM_NON_DIEGETIC_PAN + if ( ( SrcRend_p->PlayStatus == TDREND_PLAYSTATUS_PLAYING ) && ( hBinRendererTd->Listener_p->PoseUpdated || SrcSpatial_p->Updated ) ) +#else if ( hBinRendererTd->Listener_p->PoseUpdated || SrcSpatial_p->Updated ) +#endif { TDREND_SRC_REND_UpdateFiltersFromSpatialParams( hBinRendererTd, SrcRend_p, SrcSpatial_p, Src_p->hrf_left_prev, Src_p->hrf_right_prev, hrf_left_delta, hrf_right_delta, &intp_count, &Src_p->filterlength, &Src_p->itd, &Src_p->Gain, Src_p, subframe_idx ); @@ -358,6 +371,16 @@ ivas_error TDREND_GetMix( { error = TDREND_REND_RenderSourceHRFilt( Src_p, hrf_left_delta, hrf_right_delta, intp_count, output_buf, subframe_length ); } + +#ifdef ISM_NON_DIEGETIC_PAN + if ( ( SrcRend_p->InputAvailable == TRUE ) && ( SrcRend_p->PlayStatus == TDREND_PLAYSTATUS_PLAYING_NON_DIEGETIC ) ) + { + pan_left = ( SrcSpatial_p->Pos_p[1] + 1.f ) * 0.5f; + pan_right = 1.f - pan_left; + v_multc_acc( &Src_p->InputFrame_p[subframe_idx * subframe_length], pan_left, output_buf[0], subframe_length ); + v_multc_acc( &Src_p->InputFrame_p[subframe_idx * subframe_length], pan_right, output_buf[1], subframe_length ); + } +#endif } /* Populate output variable */ @@ -442,8 +465,23 @@ void TDREND_Update_object_positions( TDREND_MIX_SRC_SetPos( hBinRendererTd, nS, Pos ); TDREND_MIX_SRC_SetDirAtten( hBinRendererTd, nS, DirAtten_p ); - TDREND_MIX_SRC_SetPlayState( hBinRendererTd, nS, TDREND_PLAYSTATUS_PLAYING ); +#ifdef ISM_NON_DIEGETIC_PAN + if ( hIsmMetaData[nS]->non_diegetic_flag ) + { + Pos[0] = 0; + Pos[1] = hIsmMetaData[nS]->azimuth / 90.f; + Pos[2] = 0; + TDREND_MIX_SRC_SetPos( hBinRendererTd, nS, Pos ); + TDREND_MIX_SRC_SetPlayState( hBinRendererTd, nS, TDREND_PLAYSTATUS_PLAYING_NON_DIEGETIC ); + } + else + { + TDREND_MIX_SRC_SetPlayState( hBinRendererTd, nS, TDREND_PLAYSTATUS_PLAYING ); + } +#else + TDREND_MIX_SRC_SetPlayState( hBinRendererTd, nS, TDREND_PLAYSTATUS_PLAYING ); +#endif TDREND_MIX_SRC_SetDir( hBinRendererTd, nS, Dir ); } } @@ -618,6 +656,9 @@ ivas_error ivas_td_binaural_renderer_ext( hIsmMetaData[0]->yaw = currentPos->yaw; hIsmMetaData[0]->pitch = currentPos->pitch; hIsmMetaData[0]->radius = currentPos->radius; +#ifdef ISM_NON_DIEGETIC_PAN + hIsmMetaData[0]->non_diegetic_flag = currentPos->non_diegetic_flag; +#endif } if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, headRotData->headRotEnabled, diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index c091ed15ff0ec137e7b184a027b6cfab9ca7c2f3..81e2ce5bba4305c5ddf72763e8044439f0f62d62 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -118,6 +118,10 @@ typedef struct CREND_WRAPPER_HANDLE crendWrapper; REVERB_HANDLE hReverb; rotation_matrix rot_mat_prev; +#ifdef NON_DIEGETIC_PAN + int16_t nonDiegeticPan; + float nonDiegeticPanGain; +#endif } input_ism; typedef struct @@ -144,6 +148,10 @@ typedef struct CREND_WRAPPER_HANDLE crendWrapper; REVERB_HANDLE hReverb; rotation_gains rot_gains_prev; +#ifdef NON_DIEGETIC_PAN + int16_t nonDiegeticPan; + float nonDiegeticPanGain; +#endif lfe_routing lfeRouting; } input_mc; @@ -1010,6 +1018,9 @@ static IVAS_REND_AudioObjectPosition defaultObjectPosition( pos.radius = 1.0f; pos.yaw = 0.0f; pos.pitch = 0.0f; +#ifdef ISM_NON_DIEGETIC_PAN + pos.non_diegetic_flag = 0; +#endif return pos; } @@ -1615,7 +1626,18 @@ static ivas_error updateMcPanGainsForMcOut( inputMc->base.inConfig == IVAS_REND_AUDIO_CONFIG_MONO || inputMc->base.inConfig == IVAS_REND_AUDIO_CONFIG_LS_CUSTOM ) { - error = initMcPanGainsWithEfap( inputMc, outConfig ); +#ifdef NON_DIEGETIC_PAN + if ( ( inputMc->base.inConfig == IVAS_REND_AUDIO_CONFIG_MONO ) && ( inputMc->nonDiegeticPan ) ) + { + inputMc->panGains[0][0] = ( inputMc->nonDiegeticPanGain + 1.f ) * 0.5f; + inputMc->panGains[0][1] = 1.f - inputMc->panGains[0][0]; + error = IVAS_ERR_OK; + } + else +#endif + { + error = initMcPanGainsWithEfap( inputMc, outConfig ); + } } else if ( outConfig == IVAS_REND_AUDIO_CONFIG_MONO ) { @@ -2635,7 +2657,14 @@ static void clearInputMasa( ivas_error IVAS_REND_Open( IVAS_REND_HANDLE *phIvasRend, const int32_t outputSampleRate, - const IVAS_REND_AudioConfig outConfig ) +#ifdef NON_DIEGETIC_PAN + const IVAS_REND_AudioConfig outConfig, + const int16_t nonDiegeticPan, + const float nonDiegeticPanGain +#else + const IVAS_REND_AudioConfig outConfig +#endif +) { int16_t i; IVAS_REND_HANDLE hIvasRend; @@ -2705,6 +2734,10 @@ ivas_error IVAS_REND_Open( hIvasRend->inputsIsm[i].crendWrapper = NULL; hIvasRend->inputsIsm[i].hReverb = NULL; hIvasRend->inputsIsm[i].tdRendWrapper.hBinRendererTd = NULL; +#ifdef NON_DIEGETIC_PAN + hIvasRend->inputsIsm[i].nonDiegeticPan = nonDiegeticPan; + hIvasRend->inputsIsm[i].nonDiegeticPanGain = nonDiegeticPanGain; +#endif } for ( i = 0; i < RENDERER_MAX_MC_INPUTS; ++i ) @@ -2714,6 +2747,10 @@ ivas_error IVAS_REND_Open( hIvasRend->inputsMc[i].crendWrapper = NULL; hIvasRend->inputsMc[i].hReverb = NULL; hIvasRend->inputsMc[i].tdRendWrapper.hBinRendererTd = NULL; +#ifdef NON_DIEGETIC_PAN + hIvasRend->inputsMc[i].nonDiegeticPan = nonDiegeticPan; + hIvasRend->inputsMc[i].nonDiegeticPanGain = nonDiegeticPanGain; +#endif } for ( i = 0; i < RENDERER_MAX_SBA_INPUTS; ++i ) @@ -3789,6 +3826,9 @@ int16_t IVAS_REND_FeedRenderConfig( mvr2r( renderConfig.room_acoustics.pFc_input, hRenderConfig->roomAcoustics.pFc_input, CLDFB_NO_CHANNELS_MAX ); mvr2r( renderConfig.room_acoustics.pAcoustic_rt60, hRenderConfig->roomAcoustics.pAcoustic_rt60, CLDFB_NO_CHANNELS_MAX ); mvr2r( renderConfig.room_acoustics.pAcoustic_dsr, hRenderConfig->roomAcoustics.pAcoustic_dsr, CLDFB_NO_CHANNELS_MAX ); +#ifdef ISM_NON_DIEGETIC_PAN + mvr2r( renderConfig.directivity, hRenderConfig->directivity, 3 ); +#endif return IVAS_ERR_OK; } @@ -4489,17 +4529,30 @@ static ivas_error renderIsmToMc( /* TODO(sgi): Possible optimization: less processing needed if position didn't change */ if ( *ismInput->base.ctx.pOutConfig == IVAS_REND_AUDIO_CONFIG_STEREO ) { - set_zero( currentPanGains, 16 ); - ivas_ism_get_stereo_gains( ismInput->currentPos.azimuth, - ismInput->currentPos.elevation, - ¤tPanGains[0], - ¤tPanGains[1] ); - - set_zero( previousPanGains, 16 ); - ivas_ism_get_stereo_gains( ismInput->previousPos.azimuth, - ismInput->previousPos.elevation, - &previousPanGains[0], - &previousPanGains[1] ); +#ifdef NON_DIEGETIC_PAN + if ( ismInput->nonDiegeticPan ) + { + previousPanGains[0] = currentPanGains[0] = ( ismInput->nonDiegeticPanGain + 1.f ) * 0.5f; + previousPanGains[1] = currentPanGains[1] = 1.f - currentPanGains[0]; + error = IVAS_ERR_OK; + } + else + { +#endif + set_zero( currentPanGains, 16 ); + ivas_ism_get_stereo_gains( ismInput->currentPos.azimuth, + ismInput->currentPos.elevation, + ¤tPanGains[0], + ¤tPanGains[1] ); + + set_zero( previousPanGains, 16 ); + ivas_ism_get_stereo_gains( ismInput->previousPos.azimuth, + ismInput->previousPos.elevation, + &previousPanGains[0], + &previousPanGains[1] ); +#ifdef NON_DIEGETIC_PAN + } +#endif } else { diff --git a/lib_rend/lib_rend.h b/lib_rend/lib_rend.h index 1e9d41fd38c1c49044b537b6d5826ab3f520acb3..f9b2781110875e8d03da2a8e4d7f4c206e0b1d8a 100644 --- a/lib_rend/lib_rend.h +++ b/lib_rend/lib_rend.h @@ -134,7 +134,13 @@ typedef uint16_t IVAS_REND_InputId; ivas_error IVAS_REND_Open( IVAS_REND_HANDLE *phIvasRend, /* i/o: Pointer to renderer handle */ const int32_t outputSampleRate, /* i : output sampling rate */ +#ifdef NON_DIEGETIC_PAN + const IVAS_REND_AudioConfig outConfig, /* i : output audio config */ + const int16_t nonDiegeticPan, /* i : non-diegetic object flag */ + const float nonDiegeticPanGain /* i : non-diegetic panning gain */ +#else const IVAS_REND_AudioConfig outConfig /* i : output audio config */ +#endif ); /* Note: this will reset custom LFE routings set for any MC input */ diff --git a/lib_util/cmdln_parser.c b/lib_util/cmdln_parser.c index 200b8c86e9bf77bf6c38f39b0f444e4da097e59b..a239412f25f914b497ae1e053b1c94853f8fc1d5 100644 --- a/lib_util/cmdln_parser.c +++ b/lib_util/cmdln_parser.c @@ -31,6 +31,7 @@ *******************************************************************************************************/ #include "cmdln_parser.h" +#include "cmdl_tools.h" #include #include #include @@ -119,7 +120,11 @@ static int16_t initOpts( static int8_t stringLooksLikeOption( const char *str ) { +#ifdef NON_DIEGETIC_PAN + if ( ( str[0] == '-' ) && is_number( str ) == false ) +#else if ( str[0] == '-' ) +#endif { return 1; } @@ -130,7 +135,11 @@ static int8_t stringLooksLikeOption( static const char *stringToOptionName( const char *str ) { +#ifdef NON_DIEGETIC_PAN + while ( ( *str == '-' ) && ( ( str[1] != '0' ) || ( str[1] != '1' ) ) ) +#else while ( *str == '-' ) +#endif { ++str; } @@ -268,7 +277,7 @@ static const char *getBasename( static int32_t totalOptionNameLength( const OptionProps opt ) { - return strlen( opt.match ) + strlen( opt.matchShort ); + return (int32_t) ( strlen( opt.match ) + strlen( opt.matchShort ) ); } static void printWhitespace( diff --git a/lib_util/ism_file_reader.c b/lib_util/ism_file_reader.c index fa449512e403dc3e49634206a26b2d15e894b96f..add6a85a7a5534fcd03af8b90fea19ae3cbd6a89 100644 --- a/lib_util/ism_file_reader.c +++ b/lib_util/ism_file_reader.c @@ -35,10 +35,14 @@ #include #include - -#define META_LINE_LENGTH 200 /* max number of characters at one line of metadata input/output file */ -#define NUM_ISM_METADATA_PER_LINE 7 /* Number of ISM metadata per line in a metadata file */ -#define NUM_MIN_ISM_METADATA 2 /* Minimum number of metadata parameters (azimuth and elevation) */ +#define META_LINE_LENGTH 200 /* max number of characters at one line of metadata input/output file */ +#ifdef ISM_NON_DIEGETIC_PAN +#define NUM_ISM_METADATA_PER_LINE 8 /* Number of ISM metadata per line in a metadata file */ +#define NUM_MIN_ISM_METADATA 1 /* Minimum number of metadata parameters (azimuth) */ +#else +#define NUM_ISM_METADATA_PER_LINE 7 /* Number of ISM metadata per line in a metadata file */ +#define NUM_MIN_ISM_METADATA 2 /* Minimum number of metadata parameters (azimuth and elevation) */ +#endif struct IsmFileReader @@ -96,7 +100,11 @@ ivas_error IsmFileReader_readNextFrame( { char char_buff[META_LINE_LENGTH]; float meta_prm[NUM_ISM_METADATA_PER_LINE]; +#ifdef ISM_NON_DIEGETIC_PAN + const float meta_prm_default[NUM_ISM_METADATA_PER_LINE] = { 0.0f, 0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f }; +#else const float meta_prm_default[NUM_ISM_METADATA_PER_LINE] = { 0.0f, 0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f }; +#endif char *char_ptr; int16_t i; FILE *file; @@ -142,7 +150,6 @@ ivas_error IsmFileReader_readNextFrame( return IVAS_ERR_ISM_FILE_READER_INVALID_METADATA_FORMAT; } - ismMetadata->azimuth = meta_prm[0]; ismMetadata->elevation = meta_prm[1]; ismMetadata->radius = meta_prm[2]; @@ -150,8 +157,18 @@ ivas_error IsmFileReader_readNextFrame( ismMetadata->gainFactor = meta_prm[4]; ismMetadata->yaw = meta_prm[5]; ismMetadata->pitch = meta_prm[6]; +#ifdef ISM_NON_DIEGETIC_PAN + ismMetadata->non_diegetic_flag = (int16_t) meta_prm[7]; +#endif /* verify whether the read metadata values are in an expected range */ +#ifdef ISM_NON_DIEGETIC_PAN + if ( ( ismMetadata->non_diegetic_flag ) != 0 && ( ismMetadata->non_diegetic_flag != 1 ) ) + { + return IVAS_ERR_ISM_INVALID_METADATA_VALUE; + } +#endif + if ( ismMetadata->azimuth > 180 || ismMetadata->azimuth < -180 ) { return IVAS_ERR_ISM_INVALID_METADATA_VALUE; diff --git a/lib_util/ism_file_writer.c b/lib_util/ism_file_writer.c index 468ab0443ee1f9b9842713cd1642ce0a48cf672c..18571aa1504e536ad6a3722196d51287ead87720 100644 --- a/lib_util/ism_file_writer.c +++ b/lib_util/ism_file_writer.c @@ -63,7 +63,7 @@ ivas_error IsmFileWriter_open( strncpy( metadata_filename_loc, filePathWav, sizeof( metadata_filename_loc ) - 1 ); snprintf( ext_meta, sizeof( ext_meta ), ".%d.csv", obj_num ); - const int32_t maxNumCharactersToAppend = (int32_t) sizeof( metadata_filename_loc ) - strlen( metadata_filename_loc ) - 1; + const int32_t maxNumCharactersToAppend = (int32_t) ( sizeof( metadata_filename_loc ) - strlen( metadata_filename_loc ) - 1 ); strncat( metadata_filename_loc, ext_meta, maxNumCharactersToAppend ); strcpy( filePath, metadata_filename_loc ); @@ -114,8 +114,11 @@ ivas_error IsmFileWriter_writeFrame( file = ismWriter->file; /* IVAS_fmToDo: work in progress; currently position_azimuth, position_elevation, position_radius, spread, gain_factor */ +#ifdef ISM_NON_DIEGETIC_PAN + snprintf( char_buff, sizeof( char_buff ), "%+07.2f,%+06.2f,%05.2f,%06.2f,%04.2f,%+07.2f,%+06.2f,%d\n", ismMetadata.azimuth, ismMetadata.elevation, ismMetadata.radius, ismMetadata.spread, ismMetadata.gainFactor, ismMetadata.yaw, ismMetadata.pitch, ismMetadata.non_diegetic_flag ); +#else snprintf( char_buff, sizeof( char_buff ), "%+07.2f,%+06.2f,%05.2f,%06.2f,%04.2f,%+07.2f,%+06.2f\n", ismMetadata.azimuth, ismMetadata.elevation, ismMetadata.radius, ismMetadata.spread, ismMetadata.gainFactor, ismMetadata.yaw, ismMetadata.pitch ); - +#endif if ( file ) { fputs( char_buff, file ); diff --git a/readme.txt b/readme.txt index 397de57438376dc40fddf2b0560bad76291488be..21a5a19e7940665f7cf537520a0cf248466d22e7 100644 --- a/readme.txt +++ b/readme.txt @@ -265,8 +265,8 @@ Options: -rvf File : Reference vector specified by external trajectory file works only in combination with '-otr ref_vec' and 'ref_vec_lev' modes -render_config File : Renderer configuration option File --no_diegetic_pan : panning mono non-diegetic sound to stereo -1<= pan <=1, - left or l or 1->left, right or r or -1->right, center or c or 0->middle +-non_diegetic_pan P : panning mono non-diegetic sound to stereo -90<= P <=90, + left or l or 90->left, right or r or -90->right, center or c or 0->middle -q : Quiet mode, no frame counter default is deactivated @@ -384,25 +384,33 @@ stvISM4.csv These are comma separated files (csv) which indicate the per object position in the format: -frame azimuth, elevation, radius, spread, gain, yaw, pitch +frame azimuth, elevation, radius, spread, gain, yaw, pitch, non-diegetic with the following meaning: -| Parameter | format, value range | meaning ------------------------------------------------------------------------------------ -| azimuth | float, [-180,180[ | azimuth; positive indicates left ------------------------------------------------------------------------------------ -| elevation | float, [-90,90] | elevation; positive indicates up ------------------------------------------------------------------------------------ -| radius | float, [0, 15.75] | radius (extended metadata) ------------------------------------------------------------------------------------ -| spread | float, [0,360] | spread in angles from 0...360 deg; default: 0 ------------------------------------------------------------------------------------ -| gain | float, [0,1] | gain; default: 1 ------------------------------------------------------------------------------------ -| yaw | float, [-180,180[ | yaw (extended metadata); positive indicates left ------------------------------------------------------------------------------------ -| pitch | float, [-90,90] | pitch (extended metadata); positive indicates up ------------------------------------------------------------------------------------ +| Parameter | format, value range | meaning +--------------------------------------------------------------------------------------------------- +| azimuth | float, [-180,180] | azimuth; positive indicates left; default: 0 +--------------------------------------------------------------------------------------------------- +| elevation | float, [-90,90] | elevation; positive indicates up; default: 0 +--------------------------------------------------------------------------------------------------- +| radius | float, [0, 15.75] | radius (extended metadata); default: 1 +--------------------------------------------------------------------------------------------------- +| spread | float, [0,360] | spread in angles from 0...360 deg; default: 0 +--------------------------------------------------------------------------------------------------- +| gain | float, [0,1] | gain; default: 1 +--------------------------------------------------------------------------------------------------- +| yaw | float, [-180,180] | yaw (extended metadata); positive indicates left; default: 0 +--------------------------------------------------------------------------------------------------- +| pitch | float, [-90,90] | pitch (extended metadata); positive indicates up; default: 0 +--------------------------------------------------------------------------------------------------- +| non-diegetic | float*, [0 1] | Flag for activation of non-diegetic rendering; default: 0 +| | if Flag is set to 1, panning gain is specified by azimuth. +| | Value between [-90,90], 90 left, -90 right, 0 center +--------------------------------------------------------------------------------------------------- +*Read as float value for convenience, but used as an integer flag internally. + +The metadata reader accepts 1-8 values specified per line. If a value is not specified, the default +value is assumed. For the HRTF filter File option, external HRTF filter Files are available in folder /scripts/binauralRenderer_interface/binaural_renderers_hrtf_data : diff --git a/tests/renderer/constants.py b/tests/renderer/constants.py index c66f5e677c82a398ae3975fb808cbc2e1733c984..56b4d71e26de32cf12a9f16664808d5f8675a0c6 100644 --- a/tests/renderer/constants.py +++ b/tests/renderer/constants.py @@ -94,6 +94,7 @@ FORMAT_TO_FILE_SMOKETEST = { "ISM2": NCHAN_TO_FILE[2], "ISM3": NCHAN_TO_FILE[3], "ISM4": NCHAN_TO_FILE[4], + "NDP_ISM4": NCHAN_TO_FILE[4], # "ISM1": TEST_VECTOR_DIR.joinpath("spectral_test_ism1.txt"), # "ISM2": TEST_VECTOR_DIR.joinpath("spectral_test_ism2.txt"), # "ISM3": TEST_VECTOR_DIR.joinpath("spectral_test_ism3.txt"), @@ -148,6 +149,12 @@ FORMAT_TO_METADATA_FILES = { str(TESTV_DIR.joinpath("stvISM3.csv")), str(TESTV_DIR.joinpath("stvISM4.csv")), ], + "NDP_ISM4": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2_non-diegetic-pan.csv")), + str(TESTV_DIR.joinpath("stvISM3.csv")), + str(TESTV_DIR.joinpath("stvISM4.csv")), + ], "MASA1": [str(TESTV_DIR.joinpath("stv1MASA1TC48c.met"))], "MASA2": [str(TESTV_DIR.joinpath("stv2MASA2TC48c.met"))], } diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index d88d7a4a4299660e8891b395a75d3095332ac6d3..38b7bbf9e743c4f479790c58a91e782623c806ca 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -441,3 +441,20 @@ def test_metadata(test_info, in_fmt, out_fmt): out_fmt, metadata_input=TEST_VECTOR_DIR.joinpath(f"{in_fmt}.txt"), ) + + +""" non diegetic pan """ + + +@pytest.mark.parametrize("out_fmt", ["STEREO"]) +@pytest.mark.parametrize("in_fmt", ["MONO"]) +@pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) +def test_non_diegetic_pan_static(test_info, in_fmt, out_fmt, non_diegetic_pan): + run_renderer(in_fmt, out_fmt, non_diegetic_pan=non_diegetic_pan) + + +@pytest.mark.parametrize("out_fmt", ["STEREO"]) +@pytest.mark.parametrize("in_fmt", ["ISM1"]) +@pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) +def test_non_diegetic_pan_ism_static(test_info, in_fmt, out_fmt, non_diegetic_pan): + run_renderer(in_fmt, out_fmt, non_diegetic_pan=non_diegetic_pan) diff --git a/tests/renderer/test_renderer_be_comparison.py b/tests/renderer/test_renderer_be_comparison.py index d3c0be22c6f906308624bca2c9d321430d2e9479..460ded4250421168864976c3941e2c2fbab34561 100644 --- a/tests/renderer/test_renderer_be_comparison.py +++ b/tests/renderer/test_renderer_be_comparison.py @@ -203,3 +203,22 @@ def test_metadata(test_info, in_fmt, out_fmt): metadata_input=TEST_VECTOR_DIR.joinpath(f"{in_fmt}.txt"), is_comparetest=True, ) + + +""" non diegetic pan """ + + +# @pytest.mark.parametrize("out_fmt", ["STEREO"]) +# @pytest.mark.parametrize("in_fmt", ["MONO"]) +# @pytest.mark.parametrize("non_diegetic_pan", ["0", "-0.2", "0.5", "1", "-1"]) +# def test_non_diegetic_pan_static(test_info, in_fmt, out_fmt, non_diegetic_pan): +# compare_renderer_vs_mergetarget( +# test_info, in_fmt, out_fmt, non_diegetic_pan=non_diegetic_pan) + + +# @pytest.mark.parametrize("out_fmt", ["STEREO"]) +# @pytest.mark.parametrize("in_fmt", ["ISM1"]) +# @pytest.mark.parametrize("non_diegetic_pan", ["0", "-0.2", "0.5", "1", "-1"]) +# def test_non_diegetic_pan_ism_static(test_info, in_fmt, out_fmt, non_diegetic_pan): +# compare_renderer_vs_mergetarget( +# test_info, in_fmt, out_fmt, non_diegetic_pan=non_diegetic_pan) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index bbc059bbd6aeab67f7729560873548e6b7aaf4ba..356902ff47674ff111dbe7541bde3bac8e24bf4e 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -105,6 +105,7 @@ def run_renderer( metadata_input: Optional[str] = None, in_meta_files: Optional[list] = None, trj_file: Optional[str] = None, + non_diegetic_pan: Optional[str] = None, name_extension: Optional[str] = None, refrot_file: Optional[str] = None, refvec_file: Optional[str] = None, @@ -162,7 +163,11 @@ def run_renderer( in_file = FORMAT_TO_FILE[in_fmt] in_name = in_fmt - out_file = str(output_path_base.joinpath(f"{in_name}_to_{out_name}{trj_name}{refrot_name}{refvec_name}{refveclev_name}{config_name}{name_extension}.wav")) + out_file = str( + output_path_base.joinpath( + f"{in_name}_to_{out_name}{trj_name}{non_diegetic_pan}{refrot_name}{refvec_name}{refveclev_name}{config_name}{name_extension}.wav" + ) + ) cmd = RENDERER_CMD[:] cmd[2] = str(in_file) @@ -178,6 +183,8 @@ def run_renderer( if trj_file is not None: cmd.extend(["-tf", str(trj_file)]) + if non_diegetic_pan is not None: + cmd.extend(["-non_diegetic_pan", str(non_diegetic_pan)]) if refrot_file is not None: cmd.extend(["-rf", str(refrot_file)]) cmd.extend(["-otr", "ref"])