diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 816a73ab3aef264322d580aa937e20193428d51f..ee7c7ea91527c3723bf8e540f6b9aff73ba2720e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -167,6 +167,10 @@ stages: - sed -i.bak -e "s/\/\*[[:space:]]*\(#define[[:space:]]*SPLIT_REND_WITH_HEAD_ROT\)[[:space:]]*\*\//\1/g" ./lib_com/options.h - sed -i.bak -e "s/\/\/[[:space:]]*\(#define[[:space:]]*SPLIT_REND_WITH_HEAD_ROT\)/\1/g" ./lib_com/options.h +.disable-limiter: &disable-limiter +# automatically enable #define DISABLE_LIMITER in options.h, handling both /**/-comment and //-comment + - sed -i.bak -e "s/\/\*[[:space:]]*\(#define[[:space:]]*DISABLE_LIMITER\)[[:space:]]*\*\//\1/g" ./lib_com/options.h + .get-commits-behind-count: &get-commits-behind-count - echo $CI_COMMIT_SHA - echo $CI_MERGE_REQUEST_TARGET_BRANCH_NAME @@ -523,6 +527,54 @@ codec-usan: - SELF_TEST_PRM_FILE="tests/codec_be_on_mr_nonselection/test_param_file.py" <<: *sanitizer-selftest-anchor +# compare bit-exactness between 5ms and 20 on the branch +pytest-compare-20ms-and-5ms-rendering: + extends: + - .test-job-linux + - .rules-merge-request + stage: test + needs: ["build-codec-linux-cmake", "build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"] + script: + - *print-common-info + - *disable-limiter + - make clean + - make -j + ### prepare pytest + - cp IVAS_cod IVAS_cod_ref + - cp IVAS_dec IVAS_dec_ref + # create short test vectors + - python3 tests/create_short_testvectors.py + # create references + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2 + ### run pytest + - exit_code=0 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$? + - zero_errors5=$(cat report-junit-5ms.xml | grep -c 'errors="0"') || true + - zero_errors10=$(cat report-junit-10ms.xml | grep -c 'errors="0"') || true + - zero_errors=1 + - if [ $zero_errors5 != 1 ]; then echo "run error in with 5ms rendering encountered"; zero_errors=0 ; fi + - if [ $zero_errors10 != 1 ]; then echo "run error in with 10ms rendering encountered"; zero_errors=0 ; fi + - if [ $zero_errors != 1 ]; then exit $EXIT_CODE_FAIL; fi + - if [ $exit_code5 -eq 1 ]; then echo "Non-bitexact cases encountered with 5ms rendering!"; exit_code=1; fi + - if [ $exit_code10 -eq 1 ]; then echo "Non-bitexact cases encountered with 10ms rendering!"; exit_code=1; fi + - if [ $exit_code -eq 1 ]; then exit $EXIT_CODE_FAIL; fi + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + expose_as: "pytest 5ms and 10ms vs 20ms results" + paths: + - report-junit-5ms.xml + - report-5ms.html + - report-junit-10ms.xml + - report-10ms.html + reports: + junit: + - report-junit-5ms.xml + - report-junit-10ms.xml + # test renderer executable renderer-smoke-test: extends: @@ -1988,6 +2040,8 @@ coverage-test-on-main-scheduled: - python3 tests/create_short_testvectors.py - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_ref - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_cut - lcov -c -d obj -o coverage_stv.info # extract coverage of short test vectors here diff --git a/lib_com/options.h b/lib_com/options.h index 1e291c172101f95ca25b46a2e3ca5b0f5775d137..044e5489d9c3696437286dfe46a86492c2430e15 100644 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -146,6 +146,7 @@ /* keep as part of options.h */ #define BASOP_NOGLOB /* Disable global symbols in BASOPs, Overflow/Carry in BASOPs disabled, additional BASOPs in case of Overflow */ +/*#define DISABLE_LIMITER */ /* test switch for testing BE between 5ms and 20ms rendering */ /* ################## Start DEVELOPMENT switches ######################### */ diff --git a/scripts/config/self_test.prm b/scripts/config/self_test.prm index ae6c74187232632136180c8678ef7878c4419668..4e24a5b73719d8c3e1fcbbaf50404a5ca8fee10d 100644 --- a/scripts/config/self_test.prm +++ b/scripts/config/self_test.prm @@ -685,10 +685,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 ../IVAS_cod -sba 3 48000 48 testv/stv3OA48c.wav bit ../IVAS_dec 5_1_2 48 bit testv/stv3OA48c.wav_SBA_48000_48-48_5_1_2.tst -// SBA at 48 kbps, 48kHz in, 48kHz out, BINAURAL out, fr 5ms -../IVAS_cod -sba 3 48000 48 testv/stv3OA48c.wav bit -../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/stv3OA48c.wav_SBA_48000_48-48_binaural_ht_fr5.tst - // SBA at 64 kbps, 32kHz in, 32kHz out, FOA out, DTX, random FER at 5% ../IVAS_cod -dtx -sba 1 64000 32 testv/stvFOA32c.wav bit eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g192 bit_error @@ -806,10 +802,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 ../IVAS_cod -sba 3 512000 48 testv/stv3OA48c.wav bit ../IVAS_dec -t testv/headrot.csv -exof testv/headrot_case00_3000_q_combinedRotationTest.csv BINAURAL 48 bit testv/stv3OA48c.wav_SBA_512000_48-48_Binaural_Headrot_EXOF.tst -// SBA at 512 kbps, 48kHz in, 48kHz out, BINAURAL out, fr 5ms -../IVAS_cod -sba 3 512000 48 testv/stv3OA48c.wav bit -../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/stv3OA48c.wav_SBA_512000_48-48_binaural_ht_fr5.tst - // SBA FOA bitrate switching from 13.2 kbps to 192 kbps, 48kHz in, 48kHz out, BINAURAL out ../IVAS_cod -sba 1 ../scripts/switchPaths/sw_13k2_192k_50fr.bin 48 testv/stvFOA48c.wav bit ../IVAS_dec BINAURAL 48 bit testv/stvFOA48c.wav_sw_48-48_BINAURAL.tst @@ -1236,18 +1228,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 ../IVAS_cod -mc 7_1_4 160000 48 testv/stv714MC48c.wav bit ../IVAS_dec 7_1_4 48 bit testv/stv714MC48c.wav_MC714_160000_48-48_MC714.tst -// Multi-channel 7_1_4 at 160 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM out, fr 5ms -../IVAS_cod -mc 7_1_4 160000 48 testv/stv714MC48c.wav bit -../IVAS_dec -fr 5 BINAURAL_ROOM_IR 48 bit testv/stv714MC48c.wav_MC714_160000_48-48_MC_binaural_room_fr5.tst - -// Multi-channel 7_1_4 at 512 kbps, 48kHz in, 48kHz out, with headtracking, BINAURAL out, fr 5ms -../IVAS_cod -mc 7_1_4 512000 48 testv/stv714MC48c.wav bit -../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/stv714MC48c.wav_MC714_512000_48-48_MC_binaural_ht_fr5.tst - -// Multi-channel 5_1 at 512 kbps, 48kHz in, 48kHz out, BINAURAL out, fr 5ms -../IVAS_cod -mc 5_1 512000 48 testv/stv51MC48c.wav bit -../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/stv51MC48c.wav_MC51_512000_48-48_MC_binaural_ht_fr5.tst - // Multi-channel 5_1_2 at 32 kbps, 48kHz in, 48kHz out, STEREO out, random FER at 5% ../IVAS_cod -mc 5_1_2 32000 48 testv/stv512MC48c.wav bit eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g192 bit_error diff --git a/scripts/config/self_test_ltv.prm b/scripts/config/self_test_ltv.prm index 623f474e924c2fc8cff0897646d9039171c4dae0..6bdeaaf0a8f85d7470f1bf896df65d0a76e56337 100644 --- a/scripts/config/self_test_ltv.prm +++ b/scripts/config/self_test_ltv.prm @@ -685,10 +685,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 ../IVAS_cod -sba 3 48000 48 testv/ltv48_HOA3.wav bit ../IVAS_dec 5_1_2 48 bit testv/ltv48_HOA3.wav_SBA_48000_48-48_5_1_2.tst -// SBA at 48 kbps, 48kHz in, 48kHz out, BINAURAL out, fr 5ms -../IVAS_cod -sba 3 48000 48 testv/ltv48_HOA3.wav bit -../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/ltv48_HOA3.wav_SBA_48000_48-48_binaural_ht_fr5.tst - // SBA at 64 kbps, 32kHz in, 32kHz out, FOA out, DTX, random FER at 5% ../IVAS_cod -dtx -sba 1 64000 32 testv/ltv32_FOA.wav bit eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g192 bit_error @@ -806,10 +802,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 ../IVAS_cod -sba 3 512000 48 testv/ltv48_HOA3.wav bit ../IVAS_dec -t testv/headrot.csv -exof testv/headrot_case00_3000_q_combinedRotationTest.csv BINAURAL 48 bit testv/ltv48_HOA3.wav_SBA_512000_48-48_Binaural_Headrot_EXOF.tst -// SBA at 512 kbps, 48kHz in, 48kHz out, BINAURAL out, fr 5ms -../IVAS_cod -sba 3 512000 48 testv/ltv48_HOA3.wav bit -../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/ltv48_HOA3.wav_SBA_512000_48-48_binaural_ht_fr5.tst - // SBA FOA bitrate switching from 13.2 kbps to 192 kbps, 48kHz in, 48kHz out, BINAURAL out ../IVAS_cod -sba 1 ../scripts/switchPaths/sw_13k2_192k_50fr.bin 48 testv/ltv48_FOA.wav bit ../IVAS_dec BINAURAL 48 bit testv/ltv48_FOA.wav_sw_48-48_BINAURAL.tst @@ -1237,18 +1229,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 ../IVAS_cod -mc 7_1_4 160000 48 testv/ltv48_MC714.wav bit ../IVAS_dec 7_1_4 48 bit testv/ltv48_MC714.wav_MC714_160000_48-48_MC714.tst -// Multi-channel 7_1_4 at 160 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM out, fr 5ms -../IVAS_cod -mc 7_1_4 160000 48 testv/ltv48_MC714.wav bit -../IVAS_dec -fr 5 BINAURAL_ROOM_IR 48 bit testv/ltv48_MC714.wav_MC714_160000_48-48_MC_binaural_room_fr5.tst - -// Multi-channel 7_1_4 at 512 kbps, 48kHz in, 48kHz out, with headtracking, BINAURAL out, fr 5ms -../IVAS_cod -mc 7_1_4 512000 48 testv/ltv48_MC714.wav bit -../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/ltv48_MC714.wav_MC714_512000_48-48_MC_binaural_ht_fr5.tst - -// Multi-channel 5_1 at 512 kbps, 48kHz in, 48kHz out, BINAURAL out, fr 5ms -../IVAS_cod -mc 5_1 512000 48 testv/ltv48_MC51.wav bit -../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/ltv48_MC714.wav_MC51_512000_48-48_MC_binaural_ht_fr5.tst - // Multi-channel 5_1_2 at 32 kbps, 48kHz in, 48kHz out, STEREO out, random FER at 5% ../IVAS_cod -mc 5_1_2 32000 48 testv/ltv48_MC512.wav bit eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g192 bit_error diff --git a/scripts/self_test.py b/scripts/self_test.py index f3d204155bbcfc0d2d0c4ae7aae75fb8e623e072..c41e19993901b4b9a9802b117331a8c8903f49fb 100755 --- a/scripts/self_test.py +++ b/scripts/self_test.py @@ -195,6 +195,13 @@ class SelfTest(IvasScriptsCommon.IvasScript): action="store_true", default=False, ) + self.parser.add_argument( + "--dut_fr", + help="Run the decoder under test with specified render framn size", + default=20, + choices=[5, 10, 20], + type=int + ) if shutil.which("valgrind"): self.valgrind = [ "valgrind", @@ -1520,6 +1527,8 @@ class SelfTest(IvasScriptsCommon.IvasScript): enable_logging=True, logger_name="{}.testrunner".format(self.logger.name), ) + + test_runner.decoder_cmdline_options.extend(["-fr", f"{self.args['dut_fr']}"]) test_runner.set_flat_mode_list(run_dict) test_runner.run() self.logger.console(" ") diff --git a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py index 4f8872fc36d3f737266fc90cd030a0772cd76af3..75b01db3829d1e1786a2c9bd421f1c8007829713 100644 --- a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py +++ b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py @@ -109,6 +109,7 @@ def test_masa_enc_dec( output_mode, get_mld, get_mld_lim, + decoder_only, ): # Input parameters in_fs = 48 @@ -176,17 +177,20 @@ def test_masa_enc_dec( ) if update_ref == 2 or update_ref == 0: - # Encode DUT - ivas_enc( - dut_encoder_frontend, - masa_channel_count, - masa_path, - ivas_br, - in_fs, - input_audio_path, - output_bitstream_dut, - dtx, - ) + if decoder_only: + output_bitstream_dut = output_bitstream_ref + else: + # Encode DUT + ivas_enc( + dut_encoder_frontend, + masa_channel_count, + masa_path, + ivas_br, + in_fs, + input_audio_path, + output_bitstream_dut, + dtx, + ) # Decode DUT ivas_dec( @@ -256,16 +260,6 @@ def test_masa_enc_dec( record_property("MLD", "0") print("Comparison bit exact") - # remove_output( - # keep_files, - # output_bitstream_ref, - # output_bitstream_dut, - # dec_output_ref, - # dec_output_dut, - # dec_met_output_ref, - # dec_met_output_dut, - # ) - ######################################################### # -------------------- test function -------------------- diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 824608066c1f24ae317525e66b08ade8441b1f25..459964a1861ce9447c73c10356ac896e5ea37ddd 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -127,6 +127,7 @@ def convert_test_string_to_tag(test_string): @pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys())) def test_param_file_tests( record_property, + decoder_only, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, ref_encoder_frontend: EncoderFrontend, @@ -185,18 +186,19 @@ def test_param_file_tests( # -> construct bitstream filename bitstream_file = f"{testv_base}_{tag_str}.192" - encode( - dut_encoder_frontend, - ref_encoder_frontend, - reference_path, - dut_base_path, - bitrate, - sampling_rate, - testv_file, - bitstream_file, - enc_split, - update_ref, - ) + if not decoder_only: + encode( + dut_encoder_frontend, + ref_encoder_frontend, + reference_path, + dut_base_path, + bitrate, + sampling_rate, + testv_file, + bitstream_file, + enc_split, + update_ref, + ) if sba_br_switching_dtx == 1 and not keep_files: is_exist = os.path.exists(cut_file) if is_exist: @@ -230,6 +232,7 @@ def test_param_file_tests( sim_split, update_ref, rootdir, + decoder_only, ) # check for eid-xor command line @@ -255,6 +258,7 @@ def test_param_file_tests( eid_split, update_ref, rootdir, + decoder_only, ) # evaluate decoder options @@ -320,6 +324,7 @@ def test_param_file_tests( dec_split, update_ref, tracefile_dec, + decoder_only, ) if update_ref in [0, 2]: @@ -373,14 +378,19 @@ def test_param_file_tests( # remove DUT output files when test result is OK (to save disk space) if not keep_files: - os.remove(f"{dut_base_path}/param_file/enc/{bitstream_file}") os.remove(f"{dut_base_path}/param_file/dec/{output_file}") - if sim_opts != "": - os.remove(f"{dut_base_path}/param_file/enc/{testv_base}_{tag_str}.192") - os.remove(f"{dut_base_path}/param_file/enc/{netsim_trace_outfile}") - os.remove(f"{dut_base_path}/param_file/dec/{tracefile_dec}") - elif eid_opts != "": - os.remove(f"{dut_base_path}/param_file/enc/{testv_base}_{tag_str}.192") + if not decoder_only: + os.remove(f"{dut_base_path}/param_file/enc/{bitstream_file}") + if sim_opts != "": + os.remove( + f"{dut_base_path}/param_file/enc/{testv_base}_{tag_str}.192" + ) + os.remove(f"{dut_base_path}/param_file/enc/{netsim_trace_outfile}") + os.remove(f"{dut_base_path}/param_file/dec/{tracefile_dec}") + elif eid_opts != "": + os.remove( + f"{dut_base_path}/param_file/enc/{testv_base}_{tag_str}.192" + ) def encode( @@ -449,6 +459,7 @@ def simulate( sim_opts_list, update_ref, rootdir, + decoder_only, ): """ Call network simulator on REF and/or DUT encoder output. @@ -477,21 +488,28 @@ def simulate( else: assert False, f"networkSimulator_g192 not available for {platform.system()}" - if update_ref == 1 or update_ref == 2 and not os.path.exists(ref_out_file): + cmd_opts = sim_opts_list + if ( + update_ref == 1 + or (update_ref == 2 and not os.path.exists(ref_out_file)) + or decoder_only + ): # call network simulator on REF encoder output - cmd_opts = sim_opts_list cmd_opts[1] = f"{ref_out_dir}/{netsim_infile}" cmd_opts[2] = f"{ref_out_dir}/{netsim_outfile}" # ref_out_file - cmd_opts[3] = f"{ref_out_dir}/{netsim_tracefile}" - run(netsim + cmd_opts, check=False) - - if update_ref in [0, 2]: + elif update_ref in [0, 2]: # call network simulator on DUT encoder output - cmd_opts = sim_opts_list cmd_opts[1] = f"{dut_out_dir}/{netsim_infile}" cmd_opts[2] = f"{dut_out_dir}/{netsim_outfile}" # dut_out_file + + if update_ref == 1 or (update_ref == 2 and not os.path.exists(ref_out_file)): + cmd_opts[3] = f"{ref_out_dir}/{netsim_tracefile}" + cmd_opts[2] = f"{ref_out_dir}/{netsim_outfile}" # ref_out_file + elif update_ref in [0, 2]: cmd_opts[3] = f"{dut_out_dir}/{netsim_tracefile}" - run(netsim + cmd_opts, check=False) + cmd_opts[2] = f"{dut_out_dir}/{netsim_outfile}" # dut_out_file + + run(netsim + cmd_opts, check=False) def error_insertion( @@ -500,6 +518,7 @@ def error_insertion( eid_opts_list, update_ref, rootdir, + decoder_only, ): """ Call eid-xor to insert frame erasure on REF and/or DUT encoder output. @@ -522,19 +541,25 @@ def error_insertion( else: assert False, f"eid-xor not available for {platform.system()}" - if update_ref == 1 or update_ref == 2 and not os.path.exists(ref_out_file): + cmd_opts = eid_opts_list + + if ( + update_ref == 1 + or (update_ref == 2 and not os.path.exists(ref_out_file)) + or decoder_only + ): # call eid-xor on REF encoder output - cmd_opts = eid_opts_list cmd_opts[-3] = f"{ref_out_dir}/{eid_xor_infile}" - cmd_opts[-1] = f"{ref_out_dir}/{eid_xor_outfile}" # ref_out_file - run(eid_xor + cmd_opts, check=False) - - if update_ref in [0, 2]: + elif update_ref in [0, 2]: # call eid-xor on DUT encoder output - cmd_opts = eid_opts_list cmd_opts[-3] = f"{dut_out_dir}/{eid_xor_infile}" + + if update_ref == 1 or (update_ref == 2 and not os.path.exists(ref_out_file)): + cmd_opts[-1] = f"{ref_out_dir}/{eid_xor_outfile}" # ref_out_file + elif update_ref in [0, 2]: cmd_opts[-1] = f"{dut_out_dir}/{eid_xor_outfile}" # ref_out_file - run(eid_xor + cmd_opts, check=False) + + run(eid_xor + cmd_opts, check=False) def decode( @@ -549,6 +574,7 @@ def decode( dec_opts_list, update_ref, tracefile_dec, + decoder_only, ): """ Call REF and/or DUT decoder. @@ -589,6 +615,10 @@ def decode( x if x != "tracefile_dec" else f"{dut_out_dir}/{tracefile_dec}" for x in dec_opts_list ] + + if decoder_only: + dut_in_file = ref_in_file + # call DUT decoder decoder_frontend.run( output_config, diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index 3f394b0c2ac792d999cfa4191b49edb96cf75c94..4f4c1a3271ae86d8703bcafb2975dbab749d218c 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -104,6 +104,7 @@ def test_pca_enc( fs, get_mld, get_mld_lim, + decoder_only, ): pca = True tag = tag + fs + "c" @@ -114,27 +115,28 @@ def test_pca_enc( sba_order = "+1" output_config = "FOA" - # enc - sba_enc( - dut_encoder_frontend, - test_vector_path, - ref_encoder_frontend, - reference_path, - dut_base_path, - None, - tag, - fs, - ivas_br, - dtx, - None, - max_bw, - sba_order, - update_ref, - gain_flag, - keep_files, - cut_testv=True, - pca=pca, - ) + if not decoder_only: + # enc + sba_enc( + dut_encoder_frontend, + test_vector_path, + ref_encoder_frontend, + reference_path, + dut_base_path, + None, + tag, + fs, + ivas_br, + dtx, + None, + max_bw, + sba_order, + update_ref, + gain_flag, + keep_files, + cut_testv=True, + pca=pca, + ) # dec sba_dec( @@ -153,6 +155,7 @@ def test_pca_enc( update_ref, gain_flag, keep_files, + decoder_only, get_mld=get_mld, get_mld_lim=get_mld_lim, pca=pca, @@ -186,6 +189,7 @@ def test_sba_enc_system( SID, get_mld, get_mld_lim, + decoder_only, ): if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -215,28 +219,30 @@ def test_sba_enc_system( cut_gain = ".004" else: cut_gain = "1.0" - # enc - sba_enc( - dut_encoder_frontend, - test_vector_path, - ref_encoder_frontend, - reference_path, - dut_base_path, - br_switch_file_path, - tag, - fs, - ivas_br, - dtx, - SID, - max_bw, - sba_order, - update_ref, - gain_flag, - keep_files, - cut_gain=cut_gain, - create_dutenc=True, - cut_testv=True, - ) + + if not decoder_only: + # enc + sba_enc( + dut_encoder_frontend, + test_vector_path, + ref_encoder_frontend, + reference_path, + dut_base_path, + br_switch_file_path, + tag, + fs, + ivas_br, + dtx, + SID, + max_bw, + sba_order, + update_ref, + gain_flag, + keep_files, + cut_gain=cut_gain, + create_dutenc=True, + cut_testv=True, + ) # dec sba_dec( @@ -255,6 +261,7 @@ def test_sba_enc_system( update_ref, gain_flag, keep_files, + decoder_only, get_mld=get_mld, get_mld_lim=get_mld_lim, ) @@ -278,6 +285,7 @@ def test_spar_hoa2_enc_system( tag, get_mld, get_mld_lim, + decoder_only, ): fs = "48" dtx = "0" @@ -288,25 +296,26 @@ def test_spar_hoa2_enc_system( sba_order = "+2" output_config = "HOA2" - # enc - sba_enc( - dut_encoder_frontend, - test_vector_path, - ref_encoder_frontend, - reference_path, - dut_base_path, - None, - tag, - fs, - ivas_br, - dtx, - None, - max_bw, - sba_order, - update_ref, - gain_flag, - keep_files, - ) + if not decoder_only: + # enc + sba_enc( + dut_encoder_frontend, + test_vector_path, + ref_encoder_frontend, + reference_path, + dut_base_path, + None, + tag, + fs, + ivas_br, + dtx, + None, + max_bw, + sba_order, + update_ref, + gain_flag, + keep_files, + ) # dec sba_dec( @@ -325,6 +334,7 @@ def test_spar_hoa2_enc_system( update_ref, gain_flag, keep_files, + decoder_only, get_mld=get_mld, get_mld_lim=get_mld_lim, ) @@ -348,6 +358,7 @@ def test_spar_hoa3_enc_system( tag, get_mld, get_mld_lim, + decoder_only, ): fs = "48" dtx = "0" @@ -358,25 +369,26 @@ def test_spar_hoa3_enc_system( sba_order = "+3" output_config = "HOA3" - # enc - sba_enc( - dut_encoder_frontend, - test_vector_path, - ref_encoder_frontend, - reference_path, - dut_base_path, - None, - tag, - fs, - ivas_br, - dtx, - None, - max_bw, - sba_order, - update_ref, - gain_flag, - keep_files, - ) + if not decoder_only: + # enc + sba_enc( + dut_encoder_frontend, + test_vector_path, + ref_encoder_frontend, + reference_path, + dut_base_path, + None, + tag, + fs, + ivas_br, + dtx, + None, + max_bw, + sba_order, + update_ref, + gain_flag, + keep_files, + ) # dec sba_dec( @@ -395,6 +407,7 @@ def test_spar_hoa3_enc_system( update_ref, gain_flag, keep_files, + decoder_only, get_mld=get_mld, get_mld_lim=get_mld_lim, ) @@ -422,6 +435,7 @@ def test_sba_enc_BWforce_system( sample_rate_bw_idx, get_mld, get_mld_lim, + decoder_only, ): if dtx == "1" and ivas_br not in ["32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -437,26 +451,27 @@ def test_sba_enc_BWforce_system( sba_order = "+1" output_config = "FOA" - # enc - sba_enc( - dut_encoder_frontend, - test_vector_path, - ref_encoder_frontend, - reference_path, - dut_base_path, - None, - tag, - fs, - ivas_br, - dtx, - None, - bw, - sba_order, - update_ref, - gain_flag, - keep_files, - cut_testv=True, - ) + if not decoder_only: + # enc + sba_enc( + dut_encoder_frontend, + test_vector_path, + ref_encoder_frontend, + reference_path, + dut_base_path, + None, + tag, + fs, + ivas_br, + dtx, + None, + bw, + sba_order, + update_ref, + gain_flag, + keep_files, + cut_testv=True, + ) # dec sba_dec( @@ -475,6 +490,7 @@ def test_sba_enc_BWforce_system( update_ref, gain_flag, keep_files, + decoder_only, get_mld=get_mld, get_mld_lim=get_mld_lim, ) @@ -622,7 +638,7 @@ def sba_enc( with open(dut_pkt_file, "rb") as fp_in: with open(dut_pkt_file_cut, "wb") as fp_out: fr_cnt, cut_cnt = cut_from_start(fp_in, fp_out, 0, True) - if not keep_files: + if not keep_files: os.remove(dut_pkt_file) @@ -642,6 +658,7 @@ def sba_dec( update_ref, gain_flag, keep_files, + decoder_only, get_mld=False, get_mld_lim=0, pca=False, @@ -658,15 +675,15 @@ def sba_dec( if gain_flag != -1: short_tag_ext += f"_Gain{gain_flag}" if pca: - short_tag_ext += f"_pca" + short_tag_ext += "_pca" if SID == 1: - short_tag_ext += f"_SID_cut" + short_tag_ext += "_SID_cut" # to avoid conflicting names in case of parallel test execution, differentiate all cases long_tag_ext = "" if gain_flag != -1: long_tag_ext += f"_Gain{gain_flag}" if SID == 1: - long_tag_ext += f"_SID_cut" + long_tag_ext += "_SID_cut" dut_out_dir = f"{dut_base_path}/sba_bs/raw" ref_out_dir = f"{reference_path}/sba_bs/raw" @@ -689,6 +706,9 @@ def sba_dec( ) if update_ref == 0: + if decoder_only: + dut_in_pkt = ref_in_pkt + # call DUT decoder decoder_frontend.run( output_config, @@ -718,5 +738,6 @@ def sba_dec( # remove DUT output files when test result is OK (to save disk space) if not keep_files: - os.remove(dut_in_pkt) os.remove(dut_out_raw) + if not decoder_only: + os.remove(dut_in_pkt) diff --git a/tests/conftest.py b/tests/conftest.py index d667a43fea92bcf644a29569636882827e36363d..bbe16f4715b55fd154e8f0b7ed00c29e05b1b0be 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -176,12 +176,29 @@ def pytest_addoption(parser): action="store_true", default=False, ) + parser.addoption( "--create_cut", action="store_true", default=False, ) + parser.addoption( + "--dut_fr", + help="Render frame size for the DUT output.", + choices=[5, 10, 20], + type=int, + default=20, + ) + + parser.addoption( + "--decoder_only", + help="Only run decoder parts of tests in 'codec_be_on_mr_nonselection'. Use ref encoder output bitstreams.", + action="store_true", + default=False, + ) + + @pytest.fixture(scope="session", autouse=True) def update_ref(request): """ @@ -435,13 +452,14 @@ def dut_decoder_path(request) -> str: class DecoderFrontend: - def __init__(self, path, dec_type, timeout=None) -> None: + def __init__(self, path, dec_type, timeout=None, fr=20) -> None: self._path = path self._type = dec_type self.returncode = None self.stdout = None self.stderr = None self.timeout = timeout + self.fr = fr def run( self, @@ -459,6 +477,8 @@ class DecoderFrontend: if quiet_mode: command.extend(["-q"]) + command.extend(["-fr", str(self.fr)]) + if plc_file is not None: system = platform.system() @@ -486,10 +506,8 @@ class DecoderFrontend: try: if not os.path.exists(str(input_bitstream_path) + eid_output_suffix): result = run(eid_command, check=True) - except Exception as e: - print(result.stderr) - print(result.stdout) - pytest.fail(f"eid-xor operation failed!") + except Exception: + pytest.fail("eid-xor operation failed!") input_bitstream_path += eid_output_suffix @@ -550,7 +568,10 @@ def dut_decoder_frontend(dut_decoder_path, request) -> DecoderFrontend: Return a :class:`conftest.DecoderFrontend` instance as DUT for the test session. """ decoder = DecoderFrontend( - dut_decoder_path, "DUT", timeout=request.config.getoption("--testcase_timeout") + dut_decoder_path, + "DUT", + timeout=request.config.getoption("--testcase_timeout"), + fr=request.config.option.dut_fr, ) yield decoder @@ -680,6 +701,14 @@ def dut_base_path(request) -> str: return path +@pytest.fixture(scope="session", autouse=True) +def decoder_only(request) -> bool: + """ + Return value of cmdl param --decoder_only + """ + return request.config.getoption("--decoder_only") + + def pytest_configure(config): config.addinivalue_line("markers", "serial: mark test to run only in serial") config.addinivalue_line(