diff --git a/lib_com/options.h b/lib_com/options.h old mode 100644 new mode 100755 index 4b7274c87880381e2057a6b995a724f74e82449c..a892f7fcab66a5c81a5e154982ffa1294237741c --- a/lib_com/options.h +++ b/lib_com/options.h @@ -229,6 +229,8 @@ #define FIX_653_BUG_IN_SKIP_MATRIX /* Dlb: fix for issue #653, bug in the ivas_spar_get_skip_mat function*/ #define FIX_663_PARAM_ISM_EXT /* FhG: Issue 663: ParamISM EXT output improvement */ #define FIX_673_OMASA_OBJ_MD_SYNC /* Nokia: Fix issue 673 by updating metadata in the third subframe to account for audio delay. */ +#define PARAMUPMIX_BINAURAL_UPDATES /* Dlb : issue 652, MC ParamUpmix Binaural Updates */ + /* ################## End BE DEVELOPMENT switches ######################### */ diff --git a/lib_dec/ivas_init_dec.c b/lib_dec/ivas_init_dec.c index 73c66b537954810afb890aca5533b0955baff20d..2734c9e9ac4d93f69ce560826c9af8af202fb8f2 100644 --- a/lib_dec/ivas_init_dec.c +++ b/lib_dec/ivas_init_dec.c @@ -2610,7 +2610,18 @@ void ivas_init_dec_get_num_cldfb_instances( if ( st_ivas->mc_mode == MC_MODE_PARAMUPMIX && st_ivas->hDecoderConfig->output_config != AUDIO_CONFIG_MONO && st_ivas->hDecoderConfig->output_config != AUDIO_CONFIG_STEREO ) { +#ifdef PARAMUPMIX_BINAURAL_UPDATES + if ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV || st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV_ROOM || st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC ) + { + *numCldfbAnalyses = max( MC_PARAMUPMIX_MAX_INPUT_CHANS, *numCldfbAnalyses ); + } + else + { + *numCldfbAnalyses = max( MC_PARAMUPMIX_MIN_CLDFB, *numCldfbAnalyses ); + } +#else *numCldfbAnalyses = max( MC_PARAMUPMIX_MIN_CLDFB, *numCldfbAnalyses ); +#endif *numCldfbSyntheses = max( MC_PARAMUPMIX_MIN_CLDFB, *numCldfbSyntheses ); } diff --git a/lib_dec/ivas_mc_paramupmix_dec.c b/lib_dec/ivas_mc_paramupmix_dec.c index 9ae580ae3a07a06a34721ba4495287088b54e326..217d67d69358794394375112e9901dfefc54255a 100644 --- a/lib_dec/ivas_mc_paramupmix_dec.c +++ b/lib_dec/ivas_mc_paramupmix_dec.c @@ -58,7 +58,11 @@ static void ps_pred_process( MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix, float qmf_m #ifdef JBM_PARAMUPMIX static void ps_pred_process_sf( MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix, DECODER_TC_BUFFER_HANDLE hTcBuffer, float qmf_mod_re[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float qmf_mod_im[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float qmf_side_re[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float qmf_side_im[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float *param_interpol, const int16_t ch, const int16_t slots_rendered ); +#ifdef PARAMUPMIX_BINAURAL_UPDATES +static void ivas_mc_paramupmix_dec_sf( Decoder_Struct *st_ivas, float *output_f[MAX_OUTPUT_CHANNELS], int16_t slot_index_start ); +#else static void ivas_mc_paramupmix_dec_sf( Decoder_Struct *st_ivas, float *output_f[MAX_OUTPUT_CHANNELS] ); +#endif static void ivas_param_upmix_dec_decorr_subframes( Decoder_Struct *st_ivas, const int16_t nSamplesForRendering ); #endif @@ -181,9 +185,29 @@ void ivas_mc_paramupmix_dec( float *pPcm_temp[MC_PARAMUPMIX_COMBINATIONS * 2]; /* decorrelated and undecorrelated*/ int16_t noparamupmix_delay; AUDIO_CONFIG output_config; +#ifdef PARAMUPMIX_BINAURAL_UPDATES + int16_t subframeIdx, idx_in, index_slot, maxBand; +#ifdef SPLIT_REND_WITH_HEAD_ROT + float Cldfb_RealBuffer_subfr[MC_PARAMUPMIX_MAX_INPUT_CHANS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_ImagBuffer_subfr[MC_PARAMUPMIX_MAX_INPUT_CHANS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_RealBuffer_Binaural[MAX_HEAD_ROT_POSES][BINAURAL_CHANNELS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_ImagBuffer_Binaural[MAX_HEAD_ROT_POSES][BINAURAL_CHANNELS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; + int16_t idx_lfe; +#else + float Cldfb_RealBuffer_subfr[3 + MC_PARAMUPMIX_MAX_TRANSPORT_CHANS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_ImagBuffer_subfr[3 + MC_PARAMUPMIX_MAX_TRANSPORT_CHANS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_RealBuffer_Binaural[BINAURAL_CHANNELS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_ImagBuffer_Binaural[BINAURAL_CHANNELS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; +#endif + /* boxes = { 0 1 2 3 [4 6] [5 7] [8 10] [9 11] }; */ + int16_t chidx1[MC_PARAMUPMIX_COMBINATIONS] = { 0, 1, 4, 5 }; + int16_t chidx2[MC_PARAMUPMIX_COMBINATIONS] = { 2, 3, 6, 7 }; +#else #ifdef JBM_PARAMUPMIX float *p_output[MAX_OUTPUT_CHANNELS]; #endif +#endif + hMCParamUpmix = st_ivas->hMCParamUpmix; assert( hMCParamUpmix ); @@ -200,6 +224,7 @@ void ivas_mc_paramupmix_dec( { first_empty_channel = 8; /* Don't upmix */ } +#ifndef PARAMUPMIX_BINAURAL_UPDATES #ifdef JBM_PARAMUPMIX else if ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV || st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV_ROOM ) { @@ -212,6 +237,7 @@ void ivas_mc_paramupmix_dec( } ivas_binaural_cldfb( st_ivas, p_output ); } +#endif #endif else { @@ -269,40 +295,191 @@ void ivas_mc_paramupmix_dec( mvr2r( hMCParamUpmix->betas[ch], hMCParamUpmix->beta_prev[ch], IVAS_MAX_NUM_BANDS ); } - /* boxes = { 0 1 2 3 [4 6] [5 7] [8 10] [9 11] }; */ - pPcm_temp[0] = output_f[4]; - pPcm_temp[1] = output_f[6]; - pPcm_temp[2] = output_f[5]; - pPcm_temp[3] = output_f[7]; - pPcm_temp[4] = output_f[8]; - pPcm_temp[5] = output_f[10]; - pPcm_temp[6] = output_f[9]; - pPcm_temp[7] = output_f[11]; - - /* CLDFB synthesis */ - for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS * 2; ch++ ) +#ifdef PARAMUPMIX_BINAURAL_UPDATES + if ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV || st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV_ROOM ) { - for ( slot_idx = 0; slot_idx < CLDFB_NO_COL_MAX; slot_idx++ ) + maxBand = (int16_t) ( ( CLDFB_NO_CHANNELS_MAX * st_ivas->hDecoderConfig->output_Fs ) / 48000 ); + /* fastconv binaural rendering and CLDFB synthesis */ + for ( subframeIdx = 0; subframeIdx < ( CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES ); subframeIdx++ ) { - float *ptr_im[1], *ptr_re[1]; - ptr_re[0] = Cldfb_RealBuffer[ch][slot_idx]; - ptr_im[0] = Cldfb_ImagBuffer[ch][slot_idx]; + index_slot = subframeIdx * MAX_PARAM_SPATIAL_SUBFRAMES; + /* cldfb analysis of non-coupled, non-LFE channels */ + idx_in = 0; + for ( ch = 0; ch < first_empty_channel - 2 * MC_PARAMUPMIX_COMBINATIONS; ch++ ) + { + if ( st_ivas->hIntSetup.index_lfe[0] != ch ) + { + pPcm_temp[ch] = output_f[ch]; + /* slot loop for gathering the input data */ + for ( slot_idx = 0; slot_idx < MAX_PARAM_SPATIAL_SUBFRAMES; slot_idx++ ) + { + cldfbAnalysis_ts( &( pPcm_temp[ch][hMCParamUpmix->num_freq_bands * ( index_slot + slot_idx )] ), + Cldfb_RealBuffer_subfr[idx_in][slot_idx], + Cldfb_ImagBuffer_subfr[idx_in][slot_idx], + maxBand, st_ivas->cldfbAnaDec[2 * MC_PARAMUPMIX_COMBINATIONS + idx_in] ); + } + idx_in++; + } + } + /* copy and reorder cldfb analysis of coupled channels */ + for ( ch = 0; ch < MAX_PARAM_SPATIAL_SUBFRAMES; ch++ ) + { + for ( slot_idx = 0; slot_idx < MAX_PARAM_SPATIAL_SUBFRAMES; slot_idx++ ) + { + mvr2r( Cldfb_RealBuffer[chidx1[ch]][index_slot + slot_idx], Cldfb_RealBuffer_subfr[idx_in][slot_idx], CLDFB_NO_CHANNELS_MAX ); + mvr2r( Cldfb_ImagBuffer[chidx1[ch]][index_slot + slot_idx], Cldfb_ImagBuffer_subfr[idx_in][slot_idx], CLDFB_NO_CHANNELS_MAX ); + mvr2r( Cldfb_RealBuffer[chidx2[ch]][index_slot + slot_idx], Cldfb_RealBuffer_subfr[idx_in + 1][slot_idx], CLDFB_NO_CHANNELS_MAX ); + mvr2r( Cldfb_ImagBuffer[chidx2[ch]][index_slot + slot_idx], Cldfb_ImagBuffer_subfr[idx_in + 1][slot_idx], CLDFB_NO_CHANNELS_MAX ); + } + idx_in += 2; + } +#ifdef SPLIT_REND_WITH_HEAD_ROT + /*LFE handling for split rendering cases*/ + if ( ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) || + ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) + { + for ( idx_lfe = 0; idx_lfe < st_ivas->hIntSetup.num_lfe; idx_lfe++ ) + { + ch = st_ivas->hIntSetup.index_lfe[idx_lfe]; + /* slot loop for gathering the input data */ + for ( slot_idx = 0; slot_idx < MAX_PARAM_SPATIAL_SUBFRAMES; slot_idx++ ) + { + cldfbAnalysis_ts( &( output_f[ch][maxBand * index_slot] ), + Cldfb_RealBuffer_subfr[idx_in][slot_idx], + Cldfb_ImagBuffer_subfr[idx_in][slot_idx], + maxBand, st_ivas->cldfbAnaDec[idx_in] ); + } + idx_in++; + } +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( st_ivas->splitBinRend.hCldfbDataOut != NULL ) + { + for ( slot_idx = 0; slot_idx < MAX_PARAM_SPATIAL_SUBFRAMES; slot_idx++ ) + { + for ( ch = 0; ch < ( st_ivas->hIntSetup.nchan_out_woLFE + st_ivas->hIntSetup.num_lfe ); ch++ ) + { + mvr2r( Cldfb_RealBuffer_subfr[ch][slot_idx], st_ivas->splitBinRend.hCldfbDataOut->Cldfb_RealBuffer[ch][( subframeIdx * JBM_CLDFB_SLOTS_IN_SUBFRAME ) + slot_idx], maxBand ); + mvr2r( Cldfb_ImagBuffer_subfr[ch][slot_idx], st_ivas->splitBinRend.hCldfbDataOut->Cldfb_ImagBuffer[ch][( subframeIdx * JBM_CLDFB_SLOTS_IN_SUBFRAME ) + slot_idx], maxBand ); + } + } + st_ivas->splitBinRend.hCldfbDataOut->config = st_ivas->hIntSetup.output_config; + } +#endif + } +#endif + /* Implement binaural rendering */ + ivas_binRenderer( st_ivas->hBinRenderer, +#ifdef SPLIT_REND_WITH_HEAD_ROT + &st_ivas->splitBinRend.splitrend.multiBinPoseData, +#endif + st_ivas->hCombinedOrientationData, + subframeIdx, JBM_CLDFB_SLOTS_IN_SUBFRAME, +#ifdef SPLIT_REND_WITH_HEAD_ROT_DEBUG + NULL, +#endif + Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, + Cldfb_RealBuffer_subfr, + Cldfb_ImagBuffer_subfr ); - cldfbSynthesis( ptr_re, ptr_im, &( pPcm_temp[ch][hMCParamUpmix->num_freq_bands * slot_idx] ), - hMCParamUpmix->num_freq_bands, st_ivas->cldfbSynDec[ch] ); + +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) || + ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) + { + int16_t pos_idx; + for ( slot_idx = 0; slot_idx < JBM_CLDFB_SLOTS_IN_SUBFRAME; slot_idx++ ) + { + if ( st_ivas->hIntSetup.num_lfe > 0 ) + { + v_multc( Cldfb_RealBuffer_subfr[st_ivas->hIntSetup.nchan_out_woLFE][slot_idx], GAIN_LFE, Cldfb_RealBuffer_subfr[st_ivas->hIntSetup.nchan_out_woLFE][slot_idx], maxBand ); + v_multc( Cldfb_ImagBuffer_subfr[st_ivas->hIntSetup.nchan_out_woLFE][slot_idx], GAIN_LFE, Cldfb_ImagBuffer_subfr[st_ivas->hIntSetup.nchan_out_woLFE][slot_idx], maxBand ); + } + } + for ( pos_idx = 0; pos_idx < st_ivas->hBinRenderer->numPoses; pos_idx++ ) + { + for ( slot_idx = 0; slot_idx < JBM_CLDFB_SLOTS_IN_SUBFRAME; slot_idx++ ) + { + for ( ch = 0; ch < st_ivas->hDecoderConfig->nchan_out; ch++ ) + { + if ( st_ivas->hIntSetup.num_lfe > 0 ) + { + v_add( Cldfb_RealBuffer_Binaural[pos_idx][ch][slot_idx], + Cldfb_RealBuffer_subfr[st_ivas->hIntSetup.nchan_out_woLFE][slot_idx], + Cldfb_RealBuffer_Binaural[pos_idx][ch][slot_idx], + maxBand ); + v_add( Cldfb_ImagBuffer_Binaural[pos_idx][ch][slot_idx], + Cldfb_ImagBuffer_subfr[st_ivas->hIntSetup.nchan_out_woLFE][slot_idx], + Cldfb_ImagBuffer_Binaural[pos_idx][ch][slot_idx], + maxBand ); + } + mvr2r( Cldfb_RealBuffer_Binaural[pos_idx][ch][slot_idx], st_ivas->splitBinRend.hMultiBinCldfbData->Cldfb_RealBuffer_Binaural[( pos_idx * BINAURAL_CHANNELS ) + ch][( subframeIdx * JBM_CLDFB_SLOTS_IN_SUBFRAME ) + slot_idx], maxBand ); + mvr2r( Cldfb_ImagBuffer_Binaural[pos_idx][ch][slot_idx], st_ivas->splitBinRend.hMultiBinCldfbData->Cldfb_ImagBuffer_Binaural[( pos_idx * BINAURAL_CHANNELS ) + ch][( subframeIdx * JBM_CLDFB_SLOTS_IN_SUBFRAME ) + slot_idx], maxBand ); + } + } + } + } +#endif + + /* Implement CLDFB synthesis */ + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + float *RealBuffer[MAX_PARAM_SPATIAL_SUBFRAMES]; + float *ImagBuffer[MAX_PARAM_SPATIAL_SUBFRAMES]; + + for ( slot_idx = 0; slot_idx < MAX_PARAM_SPATIAL_SUBFRAMES; slot_idx++ ) + { +#ifdef SPLIT_REND_WITH_HEAD_ROT + RealBuffer[slot_idx] = Cldfb_RealBuffer_Binaural[0][ch][slot_idx]; + ImagBuffer[slot_idx] = Cldfb_ImagBuffer_Binaural[0][ch][slot_idx]; +#else + RealBuffer[slot_idx] = Cldfb_RealBuffer_Binaural[ch][slot_idx]; + ImagBuffer[slot_idx] = Cldfb_ImagBuffer_Binaural[ch][slot_idx]; +#endif + } + + cldfbSynthesis( RealBuffer, ImagBuffer, &( output_f[ch][index_slot * maxBand] ), maxBand * MAX_PARAM_SPATIAL_SUBFRAMES, st_ivas->cldfbSynDec[ch] ); + } } } - - /* adjust delay of other channels */ - noparamupmix_delay = NS2SA( output_Fs, IVAS_FB_DEC_DELAY_NS ); - for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS; ch++ ) + else { - float tmp_buf[L_SUBFRAME5MS_48k]; - mvr2r( &output_f[ch][output_frame - noparamupmix_delay], tmp_buf, noparamupmix_delay ); - mvr2r( output_f[ch], &output_f[ch][noparamupmix_delay], output_frame - noparamupmix_delay ); - mvr2r( hMCParamUpmix->pcm_delay[ch], output_f[ch], noparamupmix_delay ); - mvr2r( tmp_buf, hMCParamUpmix->pcm_delay[ch], noparamupmix_delay ); +#endif + pPcm_temp[0] = output_f[4]; + pPcm_temp[1] = output_f[6]; + pPcm_temp[2] = output_f[5]; + pPcm_temp[3] = output_f[7]; + pPcm_temp[4] = output_f[8]; + pPcm_temp[5] = output_f[10]; + pPcm_temp[6] = output_f[9]; + pPcm_temp[7] = output_f[11]; + + /* CLDFB synthesis */ + for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS * 2; ch++ ) + { + for ( slot_idx = 0; slot_idx < CLDFB_NO_COL_MAX; slot_idx++ ) + { + float *ptr_im[1], *ptr_re[1]; + ptr_re[0] = Cldfb_RealBuffer[ch][slot_idx]; + ptr_im[0] = Cldfb_ImagBuffer[ch][slot_idx]; + + cldfbSynthesis( ptr_re, ptr_im, &( pPcm_temp[ch][hMCParamUpmix->num_freq_bands * slot_idx] ), + hMCParamUpmix->num_freq_bands, st_ivas->cldfbSynDec[ch] ); + } + } + + /* adjust delay of other channels */ + noparamupmix_delay = NS2SA( output_Fs, IVAS_FB_DEC_DELAY_NS ); + for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS; ch++ ) + { + float tmp_buf[L_SUBFRAME5MS_48k]; + mvr2r( &output_f[ch][output_frame - noparamupmix_delay], tmp_buf, noparamupmix_delay ); + mvr2r( output_f[ch], &output_f[ch][noparamupmix_delay], output_frame - noparamupmix_delay ); + mvr2r( hMCParamUpmix->pcm_delay[ch], output_f[ch], noparamupmix_delay ); + mvr2r( tmp_buf, hMCParamUpmix->pcm_delay[ch], noparamupmix_delay ); + } +#ifdef PARAMUPMIX_BINAURAL_UPDATES } +#endif } for ( ch = first_empty_channel; ch < MAX_OUTPUT_CHANNELS; ch++ ) @@ -351,6 +528,9 @@ void ivas_mc_paramupmix_dec_render( int16_t slots_to_render, first_sf, last_sf, subframe_idx; uint16_t slot_size, ch; float *output_f_local[MAX_OUTPUT_CHANNELS]; +#ifdef PARAMUPMIX_BINAURAL_UPDATES + int16_t slot_index_start; +#endif MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix; hMCParamUpmix = st_ivas->hMCParamUpmix; @@ -384,12 +564,14 @@ void ivas_mc_paramupmix_dec_render( #ifdef DEBUGGING assert( slots_to_render == 0 ); #endif +#ifndef PARAMUPMIX_BINAURAL_UPDATES /* Implement binaural rendering */ if ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV || st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV_ROOM ) { ivas_binaural_cldfb_sf( st_ivas, *nSamplesRendered, slot_size, output_f_local ); } else +#endif { for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS; ch++ ) @@ -398,11 +580,19 @@ void ivas_mc_paramupmix_dec_render( mvr2r( hMCParamUpmix->beta_prev[ch], hMCParamUpmix->beta_sf[ch], IVAS_MAX_NUM_BANDS ); } +#ifdef PARAMUPMIX_BINAURAL_UPDATES + slot_index_start = 0; +#endif for ( subframe_idx = first_sf; subframe_idx < last_sf; subframe_idx++ ) { int16_t n_samples_sf = slot_size * st_ivas->hTcBuffer->subframe_nbslots[subframe_idx]; +#ifdef PARAMUPMIX_BINAURAL_UPDATES + ivas_mc_paramupmix_dec_sf( st_ivas, output_f_local, slot_index_start ); + slot_index_start += st_ivas->hTcBuffer->subframe_nbslots[subframe_idx]; +#else ivas_mc_paramupmix_dec_sf( st_ivas, output_f_local ); +#endif for ( ch = 0; ch < MAX_OUTPUT_CHANNELS; ch++ ) { output_f_local[ch] += n_samples_sf; @@ -822,6 +1012,10 @@ static void ps_pred_process_sf( static void ivas_mc_paramupmix_dec_sf( Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ float *output_f[MAX_OUTPUT_CHANNELS] /* i/o: synthesized core-coder transport channels */ +#ifdef PARAMUPMIX_BINAURAL_UPDATES + , + int16_t slot_index_start +#endif ) { int16_t i, ch, slot_idx, k; @@ -831,6 +1025,24 @@ static void ivas_mc_paramupmix_dec_sf( float Cldfb_ImagBuffer[MC_PARAMUPMIX_MAX_TRANSPORT_CHANS][JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; int16_t noparamupmix_delay, n_samples_rendered; MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix; +#ifdef PARAMUPMIX_BINAURAL_UPDATES + int16_t subframeIdx, idx_in, maxBand; +#ifdef SPLIT_REND_WITH_HEAD_ROT + float Cldfb_RealBuffer_subfr[MC_PARAMUPMIX_MAX_INPUT_CHANS][JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_ImagBuffer_subfr[MC_PARAMUPMIX_MAX_INPUT_CHANS][JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_RealBuffer_Binaural[1][BINAURAL_CHANNELS][JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_ImagBuffer_Binaural[1][BINAURAL_CHANNELS][JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + int16_t idx_lfe; +#else + float Cldfb_RealBuffer_subfr[3 + MC_PARAMUPMIX_MAX_TRANSPORT_CHANS][JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_ImagBuffer_subfr[3 + MC_PARAMUPMIX_MAX_TRANSPORT_CHANS][JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_RealBuffer_Binaural[BINAURAL_CHANNELS][JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_ImagBuffer_Binaural[BINAURAL_CHANNELS][JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; +#endif + /* boxes = { 0 1 2 3 [4 6] [5 7] [8 10] [9 11] }; */ + int16_t chidx1[MC_PARAMUPMIX_COMBINATIONS] = { 0, 1, 4, 5 }; + int16_t chidx2[MC_PARAMUPMIX_COMBINATIONS] = { 2, 3, 6, 7 }; +#endif hMCParamUpmix = st_ivas->hMCParamUpmix; assert( hMCParamUpmix ); @@ -880,54 +1092,180 @@ static void ivas_mc_paramupmix_dec_sf( } } } - /* boxes = { 0 1 2 3 [4 6] [5 7] [8 10] [9 11] }; */ - pPcm_temp[0] = output_f[4]; - pPcm_temp[1] = output_f[6]; - pPcm_temp[2] = output_f[5]; - pPcm_temp[3] = output_f[7]; - pPcm_temp[4] = output_f[8]; - pPcm_temp[5] = output_f[10]; - pPcm_temp[6] = output_f[9]; - pPcm_temp[7] = output_f[11]; - - /* CLDFB synthesis */ - for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS * 2; ch++ ) +#ifdef PARAMUPMIX_BINAURAL_UPDATES + if ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV || st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV_ROOM ) { - for ( slot_idx = 0; slot_idx < st_ivas->hTcBuffer->subframe_nbslots[st_ivas->hTcBuffer->subframes_rendered]; slot_idx++ ) + maxBand = (int16_t) ( ( CLDFB_NO_CHANNELS_MAX * st_ivas->hDecoderConfig->output_Fs ) / 48000 ); + subframeIdx = st_ivas->hTcBuffer->subframes_rendered; + /* fastconv binaural rendering and CLDFB synthesis */ + /* cldfb analysis of non-coupled, non-LFE channels */ + idx_in = 0; + for ( ch = 0; ch < MC_PARAMUPMIX_MAX_INPUT_CHANS - 2 * MC_PARAMUPMIX_COMBINATIONS; ch++ ) { - float *ptr_im[1], *ptr_re[1]; - ptr_re[0] = Cldfb_RealBuffer[ch][slot_idx]; - ptr_im[0] = Cldfb_ImagBuffer[ch][slot_idx]; - - cldfbSynthesis( ptr_re, ptr_im, &( pPcm_temp[ch][hMCParamUpmix->num_freq_bands * slot_idx] ), - hMCParamUpmix->num_freq_bands, st_ivas->cldfbSynDec[ch] ); + if ( st_ivas->hIntSetup.index_lfe[0] != ch ) + { + pPcm_temp[ch] = output_f[ch]; + /* slot loop for gathering the input data */ + for ( slot_idx = 0; slot_idx < st_ivas->hTcBuffer->subframe_nbslots[st_ivas->hTcBuffer->subframes_rendered]; slot_idx++ ) + { + cldfbAnalysis_ts( &( pPcm_temp[ch][hMCParamUpmix->num_freq_bands * slot_idx] ), + Cldfb_RealBuffer_subfr[idx_in][slot_idx], + Cldfb_ImagBuffer_subfr[idx_in][slot_idx], + maxBand, st_ivas->cldfbAnaDec[2 * MC_PARAMUPMIX_COMBINATIONS + idx_in] ); + } + idx_in++; + } } - } - /* adjust delay of other channels */ - noparamupmix_delay = NS2SA( st_ivas->hDecoderConfig->output_Fs, IVAS_FB_DEC_DELAY_NS ); - n_samples_rendered = st_ivas->hTcBuffer->subframe_nbslots[st_ivas->hTcBuffer->subframes_rendered] * hMCParamUpmix->num_freq_bands; - if ( n_samples_rendered > noparamupmix_delay ) - { - for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS; ch++ ) + /* copy and reorder cldfb analysis of coupled channels */ + for ( ch = 0; ch < MAX_PARAM_SPATIAL_SUBFRAMES; ch++ ) + { + for ( slot_idx = 0; slot_idx < MAX_PARAM_SPATIAL_SUBFRAMES; slot_idx++ ) + { + mvr2r( Cldfb_RealBuffer[chidx1[ch]][slot_idx], Cldfb_RealBuffer_subfr[idx_in][slot_idx], CLDFB_NO_CHANNELS_MAX ); + mvr2r( Cldfb_ImagBuffer[chidx1[ch]][slot_idx], Cldfb_ImagBuffer_subfr[idx_in][slot_idx], CLDFB_NO_CHANNELS_MAX ); + mvr2r( Cldfb_RealBuffer[chidx2[ch]][slot_idx], Cldfb_RealBuffer_subfr[idx_in + 1][slot_idx], CLDFB_NO_CHANNELS_MAX ); + mvr2r( Cldfb_ImagBuffer[chidx2[ch]][slot_idx], Cldfb_ImagBuffer_subfr[idx_in + 1][slot_idx], CLDFB_NO_CHANNELS_MAX ); + } + idx_in += 2; + } +#ifdef SPLIT_REND_WITH_HEAD_ROT + /*LFE handling for split rendering cases*/ + if ( ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) || + ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) { - float tmp_buf[L_SUBFRAME5MS_48k]; - mvr2r( &output_f[ch][n_samples_rendered - noparamupmix_delay], tmp_buf, noparamupmix_delay ); - mvr2r( output_f[ch], &output_f[ch][noparamupmix_delay], n_samples_rendered - noparamupmix_delay ); - mvr2r( hMCParamUpmix->pcm_delay[ch], output_f[ch], noparamupmix_delay ); - mvr2r( tmp_buf, hMCParamUpmix->pcm_delay[ch], noparamupmix_delay ); + for ( idx_lfe = 0; idx_lfe < st_ivas->hIntSetup.num_lfe; idx_lfe++ ) + { + ch = st_ivas->hIntSetup.index_lfe[idx_lfe]; + for ( slot_idx = 0; slot_idx < st_ivas->hTcBuffer->subframe_nbslots[st_ivas->hTcBuffer->subframes_rendered]; slot_idx++ ) + { + cldfbAnalysis_ts( &( output_f[ch][maxBand * slot_idx] ), + Cldfb_RealBuffer_subfr[idx_in][slot_idx], + Cldfb_ImagBuffer_subfr[idx_in][slot_idx], + maxBand, st_ivas->cldfbAnaDec[idx_in] ); + } + idx_in++; + } +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( st_ivas->splitBinRend.hCldfbDataOut != NULL ) + { + for ( slot_idx = 0; slot_idx < st_ivas->hTcBuffer->subframe_nbslots[st_ivas->hTcBuffer->subframes_rendered]; slot_idx++ ) + { + for ( ch = 0; ch < ( st_ivas->hIntSetup.nchan_out_woLFE + st_ivas->hIntSetup.num_lfe ); ch++ ) + { + mvr2r( Cldfb_RealBuffer_subfr[ch][slot_idx], st_ivas->splitBinRend.hCldfbDataOut->Cldfb_RealBuffer[ch][slot_index_start + slot_idx], maxBand ); + mvr2r( Cldfb_ImagBuffer_subfr[ch][slot_idx], st_ivas->splitBinRend.hCldfbDataOut->Cldfb_ImagBuffer[ch][slot_index_start + slot_idx], maxBand ); + } + } + st_ivas->splitBinRend.hCldfbDataOut->config = st_ivas->hIntSetup.output_config; + } +#endif + } +#endif + /* Implement binaural rendering */ + ivas_binRenderer( st_ivas->hBinRenderer, +#ifdef SPLIT_REND_WITH_HEAD_ROT + &st_ivas->splitBinRend.splitrend.multiBinPoseData, +#endif + st_ivas->hCombinedOrientationData, + subframeIdx, st_ivas->hTcBuffer->subframe_nbslots[subframeIdx], + Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, + Cldfb_RealBuffer_subfr, + Cldfb_ImagBuffer_subfr ); + +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) || + ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) + { + int16_t pos_idx; + for ( pos_idx = 0; pos_idx < st_ivas->hBinRenderer->numPoses; pos_idx++ ) + { + for ( slot_idx = 0; slot_idx < st_ivas->hTcBuffer->subframe_nbslots[subframeIdx]; slot_idx++ ) + { + for ( ch = 0; ch < st_ivas->hDecoderConfig->nchan_out; ch++ ) + { + mvr2r( Cldfb_RealBuffer_Binaural[pos_idx][ch][slot_idx], st_ivas->splitBinRend.hMultiBinCldfbData->Cldfb_RealBuffer_Binaural[( pos_idx * BINAURAL_CHANNELS ) + ch][slot_index_start + slot_idx], maxBand ); + mvr2r( Cldfb_ImagBuffer_Binaural[pos_idx][ch][slot_idx], st_ivas->splitBinRend.hMultiBinCldfbData->Cldfb_ImagBuffer_Binaural[( pos_idx * BINAURAL_CHANNELS ) + ch][slot_index_start + slot_idx], maxBand ); + } + } + } + } +#endif + + + /* Implement CLDFB synthesis */ + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + float *RealBuffer[MAX_PARAM_SPATIAL_SUBFRAMES]; + float *ImagBuffer[MAX_PARAM_SPATIAL_SUBFRAMES]; + + for ( slot_idx = 0; slot_idx < st_ivas->hTcBuffer->subframe_nbslots[subframeIdx]; slot_idx++ ) + { +#ifdef SPLIT_REND_WITH_HEAD_ROT + RealBuffer[slot_idx] = Cldfb_RealBuffer_Binaural[0][ch][slot_idx]; + ImagBuffer[slot_idx] = Cldfb_ImagBuffer_Binaural[0][ch][slot_idx]; +#else + RealBuffer[slot_idx] = Cldfb_RealBuffer_Binaural[ch][slot_idx]; + ImagBuffer[slot_idx] = Cldfb_ImagBuffer_Binaural[ch][slot_idx]; +#endif + } + + cldfbSynthesis( RealBuffer, ImagBuffer, &( output_f[ch][0] ), maxBand * st_ivas->hTcBuffer->subframe_nbslots[subframeIdx], st_ivas->cldfbSynDec[ch] ); } } else { - for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS; ch++ ) +#endif + /* boxes = { 0 1 2 3 [4 6] [5 7] [8 10] [9 11] }; */ + pPcm_temp[0] = output_f[4]; + pPcm_temp[1] = output_f[6]; + pPcm_temp[2] = output_f[5]; + pPcm_temp[3] = output_f[7]; + pPcm_temp[4] = output_f[8]; + pPcm_temp[5] = output_f[10]; + pPcm_temp[6] = output_f[9]; + pPcm_temp[7] = output_f[11]; + + /* CLDFB synthesis */ + for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS * 2; ch++ ) { - float tmp_buf[L_SUBFRAME5MS_48k]; - mvr2r( &output_f[ch][0], tmp_buf, n_samples_rendered ); - mvr2r( hMCParamUpmix->pcm_delay[ch], output_f[ch], n_samples_rendered ); - mvr2r( &hMCParamUpmix->pcm_delay[ch][n_samples_rendered], &hMCParamUpmix->pcm_delay[ch][0], noparamupmix_delay - n_samples_rendered ); - mvr2r( tmp_buf, &hMCParamUpmix->pcm_delay[ch][noparamupmix_delay - n_samples_rendered], n_samples_rendered ); + for ( slot_idx = 0; slot_idx < st_ivas->hTcBuffer->subframe_nbslots[st_ivas->hTcBuffer->subframes_rendered]; slot_idx++ ) + { + float *ptr_im[1], *ptr_re[1]; + ptr_re[0] = Cldfb_RealBuffer[ch][slot_idx]; + ptr_im[0] = Cldfb_ImagBuffer[ch][slot_idx]; + + cldfbSynthesis( ptr_re, ptr_im, &( pPcm_temp[ch][hMCParamUpmix->num_freq_bands * slot_idx] ), + hMCParamUpmix->num_freq_bands, st_ivas->cldfbSynDec[ch] ); + } } + /* adjust delay of other channels */ + noparamupmix_delay = NS2SA( st_ivas->hDecoderConfig->output_Fs, IVAS_FB_DEC_DELAY_NS ); + n_samples_rendered = st_ivas->hTcBuffer->subframe_nbslots[st_ivas->hTcBuffer->subframes_rendered] * hMCParamUpmix->num_freq_bands; + if ( n_samples_rendered > noparamupmix_delay ) + { + for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS; ch++ ) + { + float tmp_buf[L_SUBFRAME5MS_48k]; + mvr2r( &output_f[ch][n_samples_rendered - noparamupmix_delay], tmp_buf, noparamupmix_delay ); + mvr2r( output_f[ch], &output_f[ch][noparamupmix_delay], n_samples_rendered - noparamupmix_delay ); + mvr2r( hMCParamUpmix->pcm_delay[ch], output_f[ch], noparamupmix_delay ); + mvr2r( tmp_buf, hMCParamUpmix->pcm_delay[ch], noparamupmix_delay ); + } + } + else + { + for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS; ch++ ) + { + float tmp_buf[L_SUBFRAME5MS_48k]; + mvr2r( &output_f[ch][0], tmp_buf, n_samples_rendered ); + mvr2r( hMCParamUpmix->pcm_delay[ch], output_f[ch], n_samples_rendered ); + mvr2r( &hMCParamUpmix->pcm_delay[ch][n_samples_rendered], &hMCParamUpmix->pcm_delay[ch][0], noparamupmix_delay - n_samples_rendered ); + mvr2r( tmp_buf, &hMCParamUpmix->pcm_delay[ch][noparamupmix_delay - n_samples_rendered], n_samples_rendered ); + } + } +#ifdef PARAMUPMIX_BINAURAL_UPDATES } +#endif st_ivas->hTcBuffer->slots_rendered += st_ivas->hTcBuffer->subframe_nbslots[st_ivas->hTcBuffer->subframes_rendered]; st_ivas->hTcBuffer->subframes_rendered++; diff --git a/lib_dec/ivas_output_config.c b/lib_dec/ivas_output_config.c index f210658d0434016f653239ce2ac19c9374181d1a..907f785c17f64fedf3f6527e928dc7f71fea9e09 100644 --- a/lib_dec/ivas_output_config.c +++ b/lib_dec/ivas_output_config.c @@ -293,7 +293,11 @@ void ivas_renderer_select( #ifdef JBM_PARAMUPMIX if ( ( st_ivas->mc_mode == MC_MODE_PARAMUPMIX ) && ( *renderer_type == RENDERER_BINAURAL_FASTCONV ) ) { +#ifndef PARAMUPMIX_BINAURAL_UPDATES *internal_config = AUDIO_CONFIG_5_1_2; +#else + *internal_config = AUDIO_CONFIG_7_1_4; +#endif } #endif }