From 664b872b650506e2e3f7f450567fca4f8f8a3ea7 Mon Sep 17 00:00:00 2001 From: Stefan Bayer Date: Mon, 22 Jan 2024 07:21:32 +0100 Subject: [PATCH 01/24] added initial version of 5ms decoding BE test --- .gitlab-ci.yml | 32 ++++++++++++++++++++++++++++++++ scripts/self_test.py | 8 ++++++++ tests/conftest.py | 18 ++++++++++++++++-- 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 27212c8646..8cb358e006 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -159,6 +159,10 @@ stages: - sed -i.bak -e "s/\/\*[[:space:]]*\(#define[[:space:]]*SPLIT_REND_WITH_HEAD_ROT\)[[:space:]]*\*\//\1/g" ./lib_com/options.h - sed -i.bak -e "s/\/\/[[:space:]]*\(#define[[:space:]]*SPLIT_REND_WITH_HEAD_ROT\)/\1/g" ./lib_com/options.h +.disable-limiter: &disable-limiter +# automatically enable #define DISABLE_LIMITER in options.h, handling both /**/-comment and //-comment + - sed -i.bak -e "s/\/\*[[:space:]]*\(#define[[:space:]]*DISABLE_LIMITER\)[[:space:]]*\*\//\1/g" ./lib_com/options.h + .get-commits-behind-count: &get-commits-behind-count - echo $CI_COMMIT_SHA - echo $CI_MERGE_REQUEST_TARGET_BRANCH_NAME @@ -505,6 +509,34 @@ codec-usan: - scripts/ref/logs/ expose_as: "usan selftest results" +# compare bit-exactness between 5ms and 20 on the branch +codec-5ms: + extends: + - .test-job-linux + - .rules-merge-request + stage: test + needs: ["build-codec-linux"] + script: + - *print-common-info + - *disable-limiter + - make clean + - make -j + ### prepare pytest + # create short test vectors + - python3 tests/create_short_testvectors.py + ### run pytest + - exit_code=0 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --dut_fr5 || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + - *merge-request-comparison-check + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + paths: + - scripts/ref/logs/ + expose_as: "pytest IVAS 5ms results" + # test renderer executable renderer-smoke-test: extends: diff --git a/scripts/self_test.py b/scripts/self_test.py index 418f19b27d..7e7a7ae760 100755 --- a/scripts/self_test.py +++ b/scripts/self_test.py @@ -193,6 +193,12 @@ class SelfTest(IvasScriptsCommon.IvasScript): action="store_true", default=False, ) + self.parser.add_argument( + "--dut_fr5", + help="Run the decoder under test with 5ms rendering", + action="store_true", + default=False, + ) if shutil.which("valgrind"): self.valgrind = [ "valgrind", @@ -1502,6 +1508,8 @@ class SelfTest(IvasScriptsCommon.IvasScript): enable_logging=True, logger_name="{}.testrunner".format(self.logger.name), ) + if self.args["dut_fr5"] is True: + test_runner.decoder_cmdline_options.extend(["-fr","5"]) test_runner.set_flat_mode_list(run_dict) test_runner.run() self.logger.console(" ") diff --git a/tests/conftest.py b/tests/conftest.py index 5f7ed04a82..8410f1f47b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -158,6 +158,11 @@ def pytest_addoption(parser): help="Run the MLD tool instead of just comparing for bitexactness", ) + parser.addoption( + "--dut_fr5", + action="store_true", + help="5ms rendering for the DUT output.", + ) @pytest.fixture(scope="session", autouse=True) def update_ref(request): @@ -393,13 +398,14 @@ def dut_decoder_path(request) -> str: class DecoderFrontend: - def __init__(self, path, dec_type, timeout=None) -> None: + def __init__(self, path, dec_type, timeout=None, fr5=None) -> None: self._path = path self._type = dec_type self.returncode = None self.stdout = None self.stderr = None self.timeout = timeout + self.fr5 = fr5 def run( self, @@ -417,6 +423,9 @@ class DecoderFrontend: if quiet_mode: command.extend(["-q"]) + if self.fr5 is not None: + command.extend(["-fr", "5"]) + if plc_file is not None: system = platform.system() @@ -503,8 +512,13 @@ def dut_decoder_frontend(dut_decoder_path, request) -> DecoderFrontend: """ Return a :class:`conftest.DecoderFrontend` instance as DUT for the test session. """ + + fr5 = None + if request.config.option.dut_fr5 is True: + fr5 = True + decoder = DecoderFrontend( - dut_decoder_path, "DUT", timeout=request.config.getoption("--testcase_timeout") + dut_decoder_path, "DUT", timeout=request.config.getoption("--testcase_timeout"), fr5=fr5 ) yield decoder -- GitLab From 4a843c11738cf22c77de2845bc654b9d2c8d0f45 Mon Sep 17 00:00:00 2001 From: Stefan Bayer Date: Mon, 22 Jan 2024 08:57:10 +0100 Subject: [PATCH 02/24] add disabled switch DISABLE_LIMITER, needed for 5ms BE tests --- lib_com/options.h | 1 + 1 file changed, 1 insertion(+) diff --git a/lib_com/options.h b/lib_com/options.h index 743d9db9d2..77c27cbd7a 100755 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -142,6 +142,7 @@ /* keep as part of options.h */ #define BASOP_NOGLOB /* Disable global symbols in BASOPs, Overflow/Carry in BASOPs disabled, additional BASOPs in case of Overflow */ +/*#define DISABLE_LIMITER */ /* test switch for testing BE between 5ms and 20ms rendering */ /* ################## Start DEVELOPMENT switches ######################### */ -- GitLab From 2fd6d18a1626724c15b2f2e5034d29e964bd6784 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 24 Jan 2024 14:51:09 +0100 Subject: [PATCH 03/24] remove needs for 5ms test job --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8cb358e006..f39a619606 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -515,7 +515,6 @@ codec-5ms: - .test-job-linux - .rules-merge-request stage: test - needs: ["build-codec-linux"] script: - *print-common-info - *disable-limiter -- GitLab From 3c3ef09352c8fc665a6e8bc774c6486c382f79f0 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 24 Jan 2024 15:03:50 +0100 Subject: [PATCH 04/24] fix artifacts and pytest usage --- .gitlab-ci.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f39a619606..33bb177e0e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -510,7 +510,7 @@ codec-usan: expose_as: "usan selftest results" # compare bit-exactness between 5ms and 20 on the branch -codec-5ms: +pytest-compare-20ms-and-5ms-rendering: extends: - .test-job-linux - .rules-merge-request @@ -521,20 +521,29 @@ codec-5ms: - make clean - make -j ### prepare pytest + - cp IVAS_cod IVAS_cod_ref + - cp IVAS_dec IVAS_dec_ref # create short test vectors - python3 tests/create_short_testvectors.py + # create references + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2 ### run pytest - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --dut_fr5 || exit_code=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --dut_fr5 || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week when: always + expose_as: "pytest 5ms vs 20ms results" paths: - - scripts/ref/logs/ - expose_as: "pytest IVAS 5ms results" + - report-junit.xml + - report.html + reports: + junit: + - report-junit.xml # test renderer executable renderer-smoke-test: -- GitLab From ae380fee109f8279c4854cb3a9d27f56733710eb Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 24 Jan 2024 16:30:44 +0100 Subject: [PATCH 05/24] define needed variable for using the comparison anchor --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a931c459a2..987b13772b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -530,6 +530,8 @@ pytest-compare-20ms-and-5ms-rendering: - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2 ### run pytest - exit_code=0 + # dummy, is needed for using the comparison anchor later + - non_be_flag=0 - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --dut_fr5 || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check -- GitLab From f54030b81bd22b3cd3efc8a855eec535ffc61fa7 Mon Sep 17 00:00:00 2001 From: Stefan Bayer Date: Mon, 19 Feb 2024 12:01:45 +0100 Subject: [PATCH 06/24] allow all render frame sizes for the decoder under test in the self test and the pytests --- .gitlab-ci.yml | 2 +- scripts/self_test.py | 13 +++++++------ tests/conftest.py | 22 +++++++++------------- 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 987b13772b..655cb1ab1d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -532,7 +532,7 @@ pytest-compare-20ms-and-5ms-rendering: - exit_code=0 # dummy, is needed for using the comparison anchor later - non_be_flag=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --dut_fr5 || exit_code=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --dut_fr 5 || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check artifacts: diff --git a/scripts/self_test.py b/scripts/self_test.py index 7e7a7ae760..5ed103089e 100755 --- a/scripts/self_test.py +++ b/scripts/self_test.py @@ -194,10 +194,11 @@ class SelfTest(IvasScriptsCommon.IvasScript): default=False, ) self.parser.add_argument( - "--dut_fr5", - help="Run the decoder under test with 5ms rendering", - action="store_true", - default=False, + "--dut_fr", + help="Run the decoder under test with specified render framn size", + default=20, + choices=[5, 10, 20], + type=int ) if shutil.which("valgrind"): self.valgrind = [ @@ -1508,8 +1509,8 @@ class SelfTest(IvasScriptsCommon.IvasScript): enable_logging=True, logger_name="{}.testrunner".format(self.logger.name), ) - if self.args["dut_fr5"] is True: - test_runner.decoder_cmdline_options.extend(["-fr","5"]) + + test_runner.decoder_cmdline_options.extend(["-fr", f"{self.args['dut_fr']}"]) test_runner.set_flat_mode_list(run_dict) test_runner.run() self.logger.console(" ") diff --git a/tests/conftest.py b/tests/conftest.py index 8410f1f47b..aee9387e17 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -159,9 +159,11 @@ def pytest_addoption(parser): ) parser.addoption( - "--dut_fr5", - action="store_true", - help="5ms rendering for the DUT output.", + "--dut_fr", + help="Render frame size for the DUT output.", + choices=[5, 10, 20], + type=int, + default=20, ) @pytest.fixture(scope="session", autouse=True) @@ -398,14 +400,14 @@ def dut_decoder_path(request) -> str: class DecoderFrontend: - def __init__(self, path, dec_type, timeout=None, fr5=None) -> None: + def __init__(self, path, dec_type, timeout=None, fr=20) -> None: self._path = path self._type = dec_type self.returncode = None self.stdout = None self.stderr = None self.timeout = timeout - self.fr5 = fr5 + self.fr = fr def run( self, @@ -423,8 +425,7 @@ class DecoderFrontend: if quiet_mode: command.extend(["-q"]) - if self.fr5 is not None: - command.extend(["-fr", "5"]) + command.extend(["-fr", str(self.fr)]) if plc_file is not None: @@ -512,13 +513,8 @@ def dut_decoder_frontend(dut_decoder_path, request) -> DecoderFrontend: """ Return a :class:`conftest.DecoderFrontend` instance as DUT for the test session. """ - - fr5 = None - if request.config.option.dut_fr5 is True: - fr5 = True - decoder = DecoderFrontend( - dut_decoder_path, "DUT", timeout=request.config.getoption("--testcase_timeout"), fr5=fr5 + dut_decoder_path, "DUT", timeout=request.config.getoption("--testcase_timeout"), fr=request.config.option.dut_fr ) yield decoder -- GitLab From 85dd6253728356125fa52b43c8926ebabd2b046a Mon Sep 17 00:00:00 2001 From: Stefan Bayer Date: Mon, 19 Feb 2024 13:06:25 +0100 Subject: [PATCH 07/24] update CI config to both test 5ms and 10ms versus 20ms --- .gitlab-ci.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 655cb1ab1d..2bcf401a85 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -532,20 +532,26 @@ pytest-compare-20ms-and-5ms-rendering: - exit_code=0 # dummy, is needed for using the comparison anchor later - non_be_flag=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --dut_fr 5 || exit_code=$? - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 || exit_code1=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 || exit_code2=$? + - zero_errors=$(cat report-junit-5ms.xml report-junit-10ms.xml | grep -c 'errors="0"') || true + - exit_code = 0 + - if [ $exit_code -eq 1] || [ $exit_code2 -eq 1]; then exit_code=1; fi - *merge-request-comparison-check artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week when: always - expose_as: "pytest 5ms vs 20ms results" + expose_as: "pytest 5ms/10ms vs 20ms results" paths: - - report-junit.xml - - report.html + - report-junit-5ms.xml + - report-5ms.html + - report-junit-10ms.xml + - report-10ms.html reports: junit: - - report-junit.xml + - report-junit-5ms.xml + - report-junit-10ms.xml # test renderer executable renderer-smoke-test: -- GitLab From 319f9b0af5d50fe6601c0a58298b19aa71f1c67b Mon Sep 17 00:00:00 2001 From: Stefan Bayer Date: Mon, 19 Feb 2024 13:11:20 +0100 Subject: [PATCH 08/24] fix CI config file problem --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d61570e78c..8ffdd5aac7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -558,7 +558,7 @@ pytest-compare-20ms-and-5ms-rendering: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week when: always - expose_as: "pytest 5ms/10ms vs 20ms results" + expose_as: "pytest 5ms and 10ms vs 20ms results" paths: - report-junit-5ms.xml - report-5ms.html -- GitLab From 006bb8f647a131962b8ab849ddd447e22a8a077c Mon Sep 17 00:00:00 2001 From: Stefan Bayer Date: Tue, 27 Feb 2024 11:45:08 +0100 Subject: [PATCH 09/24] remove explicit 5ms test cases from self_test.prm --- scripts/config/self_test.prm | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/scripts/config/self_test.prm b/scripts/config/self_test.prm index 4d7d53e22e..339f7e50a5 100644 --- a/scripts/config/self_test.prm +++ b/scripts/config/self_test.prm @@ -685,10 +685,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 ../IVAS_cod -sba 3 48000 48 testv/stv3OA48c.wav bit ../IVAS_dec 5_1_2 48 bit testv/stv3OA48c.wav_SBA_48000_48-48_5_1_2.tst -// SBA at 48 kbps, 48kHz in, 48kHz out, BINAURAL out, fr 5ms -../IVAS_cod -sba 3 48000 48 testv/stv3OA48c.wav bit -../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/stv3OA48c.wav_SBA_48000_48-48_binaural_ht_fr5.tst - // SBA at 64 kbps, 32kHz in, 32kHz out, FOA out, DTX, random FER at 5% ../IVAS_cod -dtx -sba 1 64000 32 testv/stvFOA32c.wav bit eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g192 bit_error @@ -806,10 +802,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 ../IVAS_cod -sba 3 512000 48 testv/stv3OA48c.wav bit ../IVAS_dec -t testv/headrot.csv -exof testv/headrot_case00_3000_q_combinedRotationTest.csv BINAURAL 48 bit testv/stv3OA48c.wav_SBA_512000_48-48_Binaural_Headrot_EXOF.tst -// SBA at 512 kbps, 48kHz in, 48kHz out, BINAURAL out, fr 5ms -../IVAS_cod -sba 3 512000 48 testv/stv3OA48c.wav bit -../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/stv3OA48c.wav_SBA_512000_48-48_binaural_ht_fr5.tst - // SBA FOA bitrate switching from 13.2 kbps to 192 kbps, 48kHz in, 48kHz out, BINAURAL out ../IVAS_cod -sba 1 ../scripts/switchPaths/sw_13k2_192k_50fr.bin 48 testv/stvFOA48c.wav bit ../IVAS_dec BINAURAL 48 bit testv/stvFOA48c.wav_sw_48-48_BINAURAL.tst @@ -1236,18 +1228,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 ../IVAS_cod -mc 7_1_4 160000 48 testv/stv714MC48c.wav bit ../IVAS_dec 7_1_4 48 bit testv/stv714MC48c.wav_MC714_160000_48-48_MC714.tst -// Multi-channel 7_1_4 at 160 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM out, fr 5ms -../IVAS_cod -mc 7_1_4 160000 48 testv/stv714MC48c.wav bit -../IVAS_dec -fr 5 BINAURAL_ROOM_IR 48 bit testv/stv714MC48c.wav_MC714_160000_48-48_MC_binaural_room_fr5.tst - -// Multi-channel 7_1_4 at 512 kbps, 48kHz in, 48kHz out, with headtracking, BINAURAL out, fr 5ms -../IVAS_cod -mc 7_1_4 512000 48 testv/stv714MC48c.wav bit -../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/stv714MC48c.wav_MC714_512000_48-48_MC_binaural_ht_fr5.tst - -// Multi-channel 5_1 at 512 kbps, 48kHz in, 48kHz out, BINAURAL out, fr 5ms -../IVAS_cod -mc 5_1 512000 48 testv/stv51MC48c.wav bit -../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/stv51MC48c.wav_MC51_512000_48-48_MC_binaural_ht_fr5.tst - // Multi-channel 5_1_2 at 32 kbps, 48kHz in, 48kHz out, STEREO out, random FER at 5% ../IVAS_cod -mc 5_1_2 32000 48 testv/stv512MC48c.wav bit eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g192 bit_error -- GitLab From d8c6014430cb9d84c7863c4ba341c5f3f825feb2 Mon Sep 17 00:00:00 2001 From: Stefan Bayer Date: Tue, 27 Feb 2024 12:01:04 +0100 Subject: [PATCH 10/24] fix CI --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2682a7fdf1..e86dc9458c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -553,7 +553,7 @@ pytest-compare-20ms-and-5ms-rendering: - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 || exit_code1=$? - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 || exit_code2=$? - zero_errors=$(cat report-junit-5ms.xml report-junit-10ms.xml | grep -c 'errors="0"') || true - - exit_code = 0 + - exit_code=0 - if [ $exit_code -eq 1] || [ $exit_code2 -eq 1]; then exit_code=1; fi - *merge-request-comparison-check artifacts: -- GitLab From e228d75c4137c258ba5ade7e3f4e8a14fb108357 Mon Sep 17 00:00:00 2001 From: Stefan Bayer Date: Tue, 27 Feb 2024 13:04:14 +0100 Subject: [PATCH 11/24] fix CI --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e86dc9458c..ebbf74dbe2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -553,8 +553,10 @@ pytest-compare-20ms-and-5ms-rendering: - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 || exit_code1=$? - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 || exit_code2=$? - zero_errors=$(cat report-junit-5ms.xml report-junit-10ms.xml | grep -c 'errors="0"') || true + # map to usual values + - if [ $zero_errors -eq 2 ]; then zero_errors=1; else zero_errors=0; fi - exit_code=0 - - if [ $exit_code -eq 1] || [ $exit_code2 -eq 1]; then exit_code=1; fi + - if [ $exit_code1 -eq 1] || [ $exit_code2 -eq 1]; then exit_code=1; fi - *merge-request-comparison-check artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" -- GitLab From 80e9d9f14fc472c5bde5c7f6be264127aa7414ea Mon Sep 17 00:00:00 2001 From: Stefan Bayer Date: Tue, 27 Feb 2024 13:31:34 +0100 Subject: [PATCH 12/24] improve verbosity of 5/10ms BE test --- .gitlab-ci.yml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ebbf74dbe2..aac618e440 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -548,16 +548,17 @@ pytest-compare-20ms-and-5ms-rendering: - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2 ### run pytest - exit_code=0 - # dummy, is needed for using the comparison anchor later - - non_be_flag=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 || exit_code1=$? - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 || exit_code2=$? - - zero_errors=$(cat report-junit-5ms.xml report-junit-10ms.xml | grep -c 'errors="0"') || true - # map to usual values - - if [ $zero_errors -eq 2 ]; then zero_errors=1; else zero_errors=0; fi - - exit_code=0 - - if [ $exit_code1 -eq 1] || [ $exit_code2 -eq 1]; then exit_code=1; fi - - *merge-request-comparison-check + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 || exit_code5=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 || exit_code10=$? + - zero_errors5=$(cat report-junit-5ms.xml | grep -c 'errors="0"') || true + - zero_errors10=$(cat report-junit-10ms.xml | grep -c 'errors="0"') || true + - zero_errors=1 + - if [ $zero_errors5 -eq 0 ]; then echo "run error in with 5ms rendering encountered"; zero_errors=0 ; fi + - if [ $zero_errors10 -eq 0 ]; then echo "run error in with 10ms rendering encountered"; zero_errors=0 ; fi + - if [ $zero_errors != 1 ]; then exit $EXIT_CODE_FAIL; fi + - if [ $exit_code5 -eq 1 ]; then echo "Non-bitexact cases encountered with 5ms rendering!"; exit_code=1; fi + - if [ $exit_code10 -eq 1 ]; then echo "Non-bitexact cases encountered with 10ms rendering!"; exit_code=1; fi + - if [ $exit_code -eq 1 ]; exit $EXIT_CODE_FAIL; fi artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week -- GitLab From 29e3f74b44c1af98a699b7b7095b61ab2ebc3ff7 Mon Sep 17 00:00:00 2001 From: Stefan Bayer Date: Tue, 27 Feb 2024 13:53:28 +0100 Subject: [PATCH 13/24] run 5/10ms BE test always as long as the code compiles under Linux --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aac618e440..e3ab34a85c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -533,6 +533,7 @@ pytest-compare-20ms-and-5ms-rendering: - .test-job-linux - .rules-merge-request stage: test + needs: ["build-codec-linux-cmake", "build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"] script: - *print-common-info - *disable-limiter -- GitLab From 5200e49dd7222c5663f51eeca8ae6752a0eb93f2 Mon Sep 17 00:00:00 2001 From: Stefan Bayer Date: Tue, 27 Feb 2024 14:07:11 +0100 Subject: [PATCH 14/24] fix shell commands for 5/10 ms BE test --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e3ab34a85c..1352eb875b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -554,12 +554,12 @@ pytest-compare-20ms-and-5ms-rendering: - zero_errors5=$(cat report-junit-5ms.xml | grep -c 'errors="0"') || true - zero_errors10=$(cat report-junit-10ms.xml | grep -c 'errors="0"') || true - zero_errors=1 - - if [ $zero_errors5 -eq 0 ]; then echo "run error in with 5ms rendering encountered"; zero_errors=0 ; fi - - if [ $zero_errors10 -eq 0 ]; then echo "run error in with 10ms rendering encountered"; zero_errors=0 ; fi + - if [ $zero_errors5 != 1 ]; then echo "run error in with 5ms rendering encountered"; zero_errors=0 ; fi + - if [ $zero_errors10 != 1 ]; then echo "run error in with 10ms rendering encountered"; zero_errors=0 ; fi - if [ $zero_errors != 1 ]; then exit $EXIT_CODE_FAIL; fi - if [ $exit_code5 -eq 1 ]; then echo "Non-bitexact cases encountered with 5ms rendering!"; exit_code=1; fi - if [ $exit_code10 -eq 1 ]; then echo "Non-bitexact cases encountered with 10ms rendering!"; exit_code=1; fi - - if [ $exit_code -eq 1 ]; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ]; then exit $EXIT_CODE_FAIL; fi artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week -- GitLab From f4fa0d24d474fbafe718e7c3bd017860b51407d8 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 27 Feb 2024 16:52:35 +0100 Subject: [PATCH 15/24] synch self_test.prm and self_test_ltv.prm --- scripts/config/self_test_ltv.prm | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/scripts/config/self_test_ltv.prm b/scripts/config/self_test_ltv.prm index 623f474e92..6bdeaaf0a8 100644 --- a/scripts/config/self_test_ltv.prm +++ b/scripts/config/self_test_ltv.prm @@ -685,10 +685,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 ../IVAS_cod -sba 3 48000 48 testv/ltv48_HOA3.wav bit ../IVAS_dec 5_1_2 48 bit testv/ltv48_HOA3.wav_SBA_48000_48-48_5_1_2.tst -// SBA at 48 kbps, 48kHz in, 48kHz out, BINAURAL out, fr 5ms -../IVAS_cod -sba 3 48000 48 testv/ltv48_HOA3.wav bit -../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/ltv48_HOA3.wav_SBA_48000_48-48_binaural_ht_fr5.tst - // SBA at 64 kbps, 32kHz in, 32kHz out, FOA out, DTX, random FER at 5% ../IVAS_cod -dtx -sba 1 64000 32 testv/ltv32_FOA.wav bit eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g192 bit_error @@ -806,10 +802,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 ../IVAS_cod -sba 3 512000 48 testv/ltv48_HOA3.wav bit ../IVAS_dec -t testv/headrot.csv -exof testv/headrot_case00_3000_q_combinedRotationTest.csv BINAURAL 48 bit testv/ltv48_HOA3.wav_SBA_512000_48-48_Binaural_Headrot_EXOF.tst -// SBA at 512 kbps, 48kHz in, 48kHz out, BINAURAL out, fr 5ms -../IVAS_cod -sba 3 512000 48 testv/ltv48_HOA3.wav bit -../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/ltv48_HOA3.wav_SBA_512000_48-48_binaural_ht_fr5.tst - // SBA FOA bitrate switching from 13.2 kbps to 192 kbps, 48kHz in, 48kHz out, BINAURAL out ../IVAS_cod -sba 1 ../scripts/switchPaths/sw_13k2_192k_50fr.bin 48 testv/ltv48_FOA.wav bit ../IVAS_dec BINAURAL 48 bit testv/ltv48_FOA.wav_sw_48-48_BINAURAL.tst @@ -1237,18 +1229,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 ../IVAS_cod -mc 7_1_4 160000 48 testv/ltv48_MC714.wav bit ../IVAS_dec 7_1_4 48 bit testv/ltv48_MC714.wav_MC714_160000_48-48_MC714.tst -// Multi-channel 7_1_4 at 160 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM out, fr 5ms -../IVAS_cod -mc 7_1_4 160000 48 testv/ltv48_MC714.wav bit -../IVAS_dec -fr 5 BINAURAL_ROOM_IR 48 bit testv/ltv48_MC714.wav_MC714_160000_48-48_MC_binaural_room_fr5.tst - -// Multi-channel 7_1_4 at 512 kbps, 48kHz in, 48kHz out, with headtracking, BINAURAL out, fr 5ms -../IVAS_cod -mc 7_1_4 512000 48 testv/ltv48_MC714.wav bit -../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/ltv48_MC714.wav_MC714_512000_48-48_MC_binaural_ht_fr5.tst - -// Multi-channel 5_1 at 512 kbps, 48kHz in, 48kHz out, BINAURAL out, fr 5ms -../IVAS_cod -mc 5_1 512000 48 testv/ltv48_MC51.wav bit -../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/ltv48_MC714.wav_MC51_512000_48-48_MC_binaural_ht_fr5.tst - // Multi-channel 5_1_2 at 32 kbps, 48kHz in, 48kHz out, STEREO out, random FER at 5% ../IVAS_cod -mc 5_1_2 32000 48 testv/ltv48_MC512.wav bit eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g192 bit_error -- GitLab From 66f4cdf7abcb75de309a68286f208848f6998fb2 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 28 Feb 2024 10:39:53 +0100 Subject: [PATCH 16/24] add --decoder_only option and use it in test_param_files.py --- .../test_param_file.py | 88 +++++++++++-------- tests/conftest.py | 15 ++++ 2 files changed, 66 insertions(+), 37 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 824608066c..226b0794bc 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -127,6 +127,7 @@ def convert_test_string_to_tag(test_string): @pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys())) def test_param_file_tests( record_property, + decoder_only, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, ref_encoder_frontend: EncoderFrontend, @@ -185,18 +186,19 @@ def test_param_file_tests( # -> construct bitstream filename bitstream_file = f"{testv_base}_{tag_str}.192" - encode( - dut_encoder_frontend, - ref_encoder_frontend, - reference_path, - dut_base_path, - bitrate, - sampling_rate, - testv_file, - bitstream_file, - enc_split, - update_ref, - ) + if not decoder_only: + encode( + dut_encoder_frontend, + ref_encoder_frontend, + reference_path, + dut_base_path, + bitrate, + sampling_rate, + testv_file, + bitstream_file, + enc_split, + update_ref, + ) if sba_br_switching_dtx == 1 and not keep_files: is_exist = os.path.exists(cut_file) if is_exist: @@ -230,6 +232,7 @@ def test_param_file_tests( sim_split, update_ref, rootdir, + decoder_only, ) # check for eid-xor command line @@ -255,6 +258,7 @@ def test_param_file_tests( eid_split, update_ref, rootdir, + decoder_only, ) # evaluate decoder options @@ -320,6 +324,7 @@ def test_param_file_tests( dec_split, update_ref, tracefile_dec, + decoder_only, ) if update_ref in [0, 2]: @@ -373,14 +378,15 @@ def test_param_file_tests( # remove DUT output files when test result is OK (to save disk space) if not keep_files: - os.remove(f"{dut_base_path}/param_file/enc/{bitstream_file}") os.remove(f"{dut_base_path}/param_file/dec/{output_file}") - if sim_opts != "": - os.remove(f"{dut_base_path}/param_file/enc/{testv_base}_{tag_str}.192") - os.remove(f"{dut_base_path}/param_file/enc/{netsim_trace_outfile}") - os.remove(f"{dut_base_path}/param_file/dec/{tracefile_dec}") - elif eid_opts != "": - os.remove(f"{dut_base_path}/param_file/enc/{testv_base}_{tag_str}.192") + if not decoder_only: + os.remove(f"{dut_base_path}/param_file/enc/{bitstream_file}") + if sim_opts != "": + os.remove(f"{dut_base_path}/param_file/enc/{testv_base}_{tag_str}.192") + os.remove(f"{dut_base_path}/param_file/enc/{netsim_trace_outfile}") + os.remove(f"{dut_base_path}/param_file/dec/{tracefile_dec}") + elif eid_opts != "": + os.remove(f"{dut_base_path}/param_file/enc/{testv_base}_{tag_str}.192") def encode( @@ -449,6 +455,7 @@ def simulate( sim_opts_list, update_ref, rootdir, + decoder_only, ): """ Call network simulator on REF and/or DUT encoder output. @@ -477,21 +484,20 @@ def simulate( else: assert False, f"networkSimulator_g192 not available for {platform.system()}" - if update_ref == 1 or update_ref == 2 and not os.path.exists(ref_out_file): + cmd_opts = sim_opts_list + if update_ref == 1 or (update_ref == 2 and not os.path.exists(ref_out_file)) or decoder_only: # call network simulator on REF encoder output - cmd_opts = sim_opts_list cmd_opts[1] = f"{ref_out_dir}/{netsim_infile}" - cmd_opts[2] = f"{ref_out_dir}/{netsim_outfile}" # ref_out_file - cmd_opts[3] = f"{ref_out_dir}/{netsim_tracefile}" - run(netsim + cmd_opts, check=False) - - if update_ref in [0, 2]: + elif update_ref in [0, 2]: # call network simulator on DUT encoder output - cmd_opts = sim_opts_list cmd_opts[1] = f"{dut_out_dir}/{netsim_infile}" - cmd_opts[2] = f"{dut_out_dir}/{netsim_outfile}" # dut_out_file + + if update_ref == 1 or (update_ref == 2 and not os.path.exists(ref_out_file)): + cmd_opts[3] = f"{ref_out_dir}/{netsim_tracefile}" + elif update_ref in [0, 2]: cmd_opts[3] = f"{dut_out_dir}/{netsim_tracefile}" - run(netsim + cmd_opts, check=False) + + run(netsim + cmd_opts, check=False) def error_insertion( @@ -500,6 +506,7 @@ def error_insertion( eid_opts_list, update_ref, rootdir, + decoder_only, ): """ Call eid-xor to insert frame erasure on REF and/or DUT encoder output. @@ -522,19 +529,21 @@ def error_insertion( else: assert False, f"eid-xor not available for {platform.system()}" - if update_ref == 1 or update_ref == 2 and not os.path.exists(ref_out_file): + cmd_opts = eid_opts_list + + if update_ref == 1 or (update_ref == 2 and not os.path.exists(ref_out_file)) or decoder_only: # call eid-xor on REF encoder output - cmd_opts = eid_opts_list cmd_opts[-3] = f"{ref_out_dir}/{eid_xor_infile}" - cmd_opts[-1] = f"{ref_out_dir}/{eid_xor_outfile}" # ref_out_file - run(eid_xor + cmd_opts, check=False) - - if update_ref in [0, 2]: + elif update_ref in [0, 2]: # call eid-xor on DUT encoder output - cmd_opts = eid_opts_list cmd_opts[-3] = f"{dut_out_dir}/{eid_xor_infile}" + + if update_ref == 1 or (update_ref == 2 and not os.path.exists(ref_out_file)): + cmd_opts[-1] = f"{ref_out_dir}/{eid_xor_outfile}" # ref_out_file + elif update_ref in [0, 2]: cmd_opts[-1] = f"{dut_out_dir}/{eid_xor_outfile}" # ref_out_file - run(eid_xor + cmd_opts, check=False) + + run(eid_xor + cmd_opts, check=False) def decode( @@ -549,6 +558,7 @@ def decode( dec_opts_list, update_ref, tracefile_dec, + decoder_only, ): """ Call REF and/or DUT decoder. @@ -589,6 +599,10 @@ def decode( x if x != "tracefile_dec" else f"{dut_out_dir}/{tracefile_dec}" for x in dec_opts_list ] + + if decoder_only: + dut_in_file = ref_in_file + # call DUT decoder decoder_frontend.run( output_config, diff --git a/tests/conftest.py b/tests/conftest.py index 1e54c4ed07..76572eadd4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -191,6 +191,13 @@ def pytest_addoption(parser): default=20, ) + parser.addoption( + "--decoder_only", + help="Only run decoder parts of tests in 'codec_be_on_mr_nonselection'. Use ref encoder output bitstreams.", + action="store_true", + default=False, + ) + @pytest.fixture(scope="session", autouse=True) def update_ref(request): """ @@ -692,6 +699,14 @@ def dut_base_path(request) -> str: return path +@pytest.fixture(scope="session", autouse=True) +def decoder_only(request) -> bool: + """ + Return value of cmdl param --decoder_only + """ + return request.config.getoption("--decoder_only") + + def pytest_configure(config): config.addinivalue_line("markers", "serial: mark test to run only in serial") config.addinivalue_line( -- GitLab From 81f977c7131ab434dd698b1cca0bad6977454133 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 28 Feb 2024 11:25:00 +0100 Subject: [PATCH 17/24] implement --decoder_only for masa tests --- .../test_masa_enc_dec.py | 38 ++++++++----------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py index 4f8872fc36..51fd1870e9 100644 --- a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py +++ b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py @@ -109,6 +109,7 @@ def test_masa_enc_dec( output_mode, get_mld, get_mld_lim, + decoder_only, ): # Input parameters in_fs = 48 @@ -153,7 +154,7 @@ def test_masa_enc_dec( dec_met_output_ref = None dec_met_output_dut = None - if update_ref == 2 or update_ref == 1: + if (update_ref == 2 or update_ref == 1): # Encode REF ivas_enc( ref_encoder_frontend, @@ -176,17 +177,20 @@ def test_masa_enc_dec( ) if update_ref == 2 or update_ref == 0: - # Encode DUT - ivas_enc( - dut_encoder_frontend, - masa_channel_count, - masa_path, - ivas_br, - in_fs, - input_audio_path, - output_bitstream_dut, - dtx, - ) + if decoder_only: + output_bitstream_dut = output_bitstream_ref + else: + # Encode DUT + ivas_enc( + dut_encoder_frontend, + masa_channel_count, + masa_path, + ivas_br, + in_fs, + input_audio_path, + output_bitstream_dut, + dtx, + ) # Decode DUT ivas_dec( @@ -256,16 +260,6 @@ def test_masa_enc_dec( record_property("MLD", "0") print("Comparison bit exact") - # remove_output( - # keep_files, - # output_bitstream_ref, - # output_bitstream_dut, - # dec_output_ref, - # dec_output_dut, - # dec_met_output_ref, - # dec_met_output_dut, - # ) - ######################################################### # -------------------- test function -------------------- -- GitLab From 1a3b444e1928f1e4e4f12a37091e96a36ccfea31 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 28 Feb 2024 12:15:49 +0100 Subject: [PATCH 18/24] implement --decoder_only for sba tests --- .../test_sba_bs_enc.py | 221 ++++++++++-------- 1 file changed, 119 insertions(+), 102 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index 3f394b0c2a..fb10b2974e 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -104,6 +104,7 @@ def test_pca_enc( fs, get_mld, get_mld_lim, + decoder_only, ): pca = True tag = tag + fs + "c" @@ -114,27 +115,28 @@ def test_pca_enc( sba_order = "+1" output_config = "FOA" - # enc - sba_enc( - dut_encoder_frontend, - test_vector_path, - ref_encoder_frontend, - reference_path, - dut_base_path, - None, - tag, - fs, - ivas_br, - dtx, - None, - max_bw, - sba_order, - update_ref, - gain_flag, - keep_files, - cut_testv=True, - pca=pca, - ) + if not decoder_only: + # enc + sba_enc( + dut_encoder_frontend, + test_vector_path, + ref_encoder_frontend, + reference_path, + dut_base_path, + None, + tag, + fs, + ivas_br, + dtx, + None, + max_bw, + sba_order, + update_ref, + gain_flag, + keep_files, + cut_testv=True, + pca=pca, + ) # dec sba_dec( @@ -153,6 +155,7 @@ def test_pca_enc( update_ref, gain_flag, keep_files, + decoder_only, get_mld=get_mld, get_mld_lim=get_mld_lim, pca=pca, @@ -186,6 +189,7 @@ def test_sba_enc_system( SID, get_mld, get_mld_lim, + decoder_only, ): if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -215,28 +219,30 @@ def test_sba_enc_system( cut_gain = ".004" else: cut_gain = "1.0" - # enc - sba_enc( - dut_encoder_frontend, - test_vector_path, - ref_encoder_frontend, - reference_path, - dut_base_path, - br_switch_file_path, - tag, - fs, - ivas_br, - dtx, - SID, - max_bw, - sba_order, - update_ref, - gain_flag, - keep_files, - cut_gain=cut_gain, - create_dutenc=True, - cut_testv=True, - ) + + if not decoder_only: + # enc + sba_enc( + dut_encoder_frontend, + test_vector_path, + ref_encoder_frontend, + reference_path, + dut_base_path, + br_switch_file_path, + tag, + fs, + ivas_br, + dtx, + SID, + max_bw, + sba_order, + update_ref, + gain_flag, + keep_files, + cut_gain=cut_gain, + create_dutenc=True, + cut_testv=True, + ) # dec sba_dec( @@ -255,6 +261,7 @@ def test_sba_enc_system( update_ref, gain_flag, keep_files, + decoder_only, get_mld=get_mld, get_mld_lim=get_mld_lim, ) @@ -278,6 +285,7 @@ def test_spar_hoa2_enc_system( tag, get_mld, get_mld_lim, + decoder_only, ): fs = "48" dtx = "0" @@ -288,25 +296,26 @@ def test_spar_hoa2_enc_system( sba_order = "+2" output_config = "HOA2" - # enc - sba_enc( - dut_encoder_frontend, - test_vector_path, - ref_encoder_frontend, - reference_path, - dut_base_path, - None, - tag, - fs, - ivas_br, - dtx, - None, - max_bw, - sba_order, - update_ref, - gain_flag, - keep_files, - ) + if not decoder_only: + # enc + sba_enc( + dut_encoder_frontend, + test_vector_path, + ref_encoder_frontend, + reference_path, + dut_base_path, + None, + tag, + fs, + ivas_br, + dtx, + None, + max_bw, + sba_order, + update_ref, + gain_flag, + keep_files, + ) # dec sba_dec( @@ -325,6 +334,7 @@ def test_spar_hoa2_enc_system( update_ref, gain_flag, keep_files, + decoder_only, get_mld=get_mld, get_mld_lim=get_mld_lim, ) @@ -348,6 +358,7 @@ def test_spar_hoa3_enc_system( tag, get_mld, get_mld_lim, + decoder_only, ): fs = "48" dtx = "0" @@ -358,25 +369,26 @@ def test_spar_hoa3_enc_system( sba_order = "+3" output_config = "HOA3" - # enc - sba_enc( - dut_encoder_frontend, - test_vector_path, - ref_encoder_frontend, - reference_path, - dut_base_path, - None, - tag, - fs, - ivas_br, - dtx, - None, - max_bw, - sba_order, - update_ref, - gain_flag, - keep_files, - ) + if not decoder_only: + # enc + sba_enc( + dut_encoder_frontend, + test_vector_path, + ref_encoder_frontend, + reference_path, + dut_base_path, + None, + tag, + fs, + ivas_br, + dtx, + None, + max_bw, + sba_order, + update_ref, + gain_flag, + keep_files, + ) # dec sba_dec( @@ -395,6 +407,7 @@ def test_spar_hoa3_enc_system( update_ref, gain_flag, keep_files, + decoder_only, get_mld=get_mld, get_mld_lim=get_mld_lim, ) @@ -422,6 +435,7 @@ def test_sba_enc_BWforce_system( sample_rate_bw_idx, get_mld, get_mld_lim, + decoder_only, ): if dtx == "1" and ivas_br not in ["32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -437,26 +451,27 @@ def test_sba_enc_BWforce_system( sba_order = "+1" output_config = "FOA" - # enc - sba_enc( - dut_encoder_frontend, - test_vector_path, - ref_encoder_frontend, - reference_path, - dut_base_path, - None, - tag, - fs, - ivas_br, - dtx, - None, - bw, - sba_order, - update_ref, - gain_flag, - keep_files, - cut_testv=True, - ) + if not decoder_only: + # enc + sba_enc( + dut_encoder_frontend, + test_vector_path, + ref_encoder_frontend, + reference_path, + dut_base_path, + None, + tag, + fs, + ivas_br, + dtx, + None, + bw, + sba_order, + update_ref, + gain_flag, + keep_files, + cut_testv=True, + ) # dec sba_dec( @@ -475,6 +490,7 @@ def test_sba_enc_BWforce_system( update_ref, gain_flag, keep_files, + decoder_only, get_mld=get_mld, get_mld_lim=get_mld_lim, ) @@ -642,6 +658,7 @@ def sba_dec( update_ref, gain_flag, keep_files, + decoder_only, get_mld=False, get_mld_lim=0, pca=False, @@ -693,7 +710,7 @@ def sba_dec( decoder_frontend.run( output_config, sampling_rate, - dut_in_pkt, + dut_in_pkt if not decoder_only else ref_in_pkt, dut_out_raw, ) -- GitLab From cab508e68eb1043244d1c18555f1fccd50450000 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 28 Feb 2024 12:17:48 +0100 Subject: [PATCH 19/24] format changed files --- .../test_masa_enc_dec.py | 2 +- .../test_param_file.py | 22 ++++++++++++++----- .../test_sba_bs_enc.py | 2 +- tests/conftest.py | 16 +++++++++----- 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py index 51fd1870e9..75b01db382 100644 --- a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py +++ b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py @@ -154,7 +154,7 @@ def test_masa_enc_dec( dec_met_output_ref = None dec_met_output_dut = None - if (update_ref == 2 or update_ref == 1): + if update_ref == 2 or update_ref == 1: # Encode REF ivas_enc( ref_encoder_frontend, diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 226b0794bc..0a494d7248 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -382,11 +382,15 @@ def test_param_file_tests( if not decoder_only: os.remove(f"{dut_base_path}/param_file/enc/{bitstream_file}") if sim_opts != "": - os.remove(f"{dut_base_path}/param_file/enc/{testv_base}_{tag_str}.192") + os.remove( + f"{dut_base_path}/param_file/enc/{testv_base}_{tag_str}.192" + ) os.remove(f"{dut_base_path}/param_file/enc/{netsim_trace_outfile}") os.remove(f"{dut_base_path}/param_file/dec/{tracefile_dec}") elif eid_opts != "": - os.remove(f"{dut_base_path}/param_file/enc/{testv_base}_{tag_str}.192") + os.remove( + f"{dut_base_path}/param_file/enc/{testv_base}_{tag_str}.192" + ) def encode( @@ -485,7 +489,11 @@ def simulate( assert False, f"networkSimulator_g192 not available for {platform.system()}" cmd_opts = sim_opts_list - if update_ref == 1 or (update_ref == 2 and not os.path.exists(ref_out_file)) or decoder_only: + if ( + update_ref == 1 + or (update_ref == 2 and not os.path.exists(ref_out_file)) + or decoder_only + ): # call network simulator on REF encoder output cmd_opts[1] = f"{ref_out_dir}/{netsim_infile}" elif update_ref in [0, 2]: @@ -531,7 +539,11 @@ def error_insertion( cmd_opts = eid_opts_list - if update_ref == 1 or (update_ref == 2 and not os.path.exists(ref_out_file)) or decoder_only: + if ( + update_ref == 1 + or (update_ref == 2 and not os.path.exists(ref_out_file)) + or decoder_only + ): # call eid-xor on REF encoder output cmd_opts[-3] = f"{ref_out_dir}/{eid_xor_infile}" elif update_ref in [0, 2]: @@ -599,7 +611,7 @@ def decode( x if x != "tracefile_dec" else f"{dut_out_dir}/{tracefile_dec}" for x in dec_opts_list ] - + if decoder_only: dut_in_file = ref_in_file diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index fb10b2974e..e088196ff1 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -638,7 +638,7 @@ def sba_enc( with open(dut_pkt_file, "rb") as fp_in: with open(dut_pkt_file_cut, "wb") as fp_out: fr_cnt, cut_cnt = cut_from_start(fp_in, fp_out, 0, True) - if not keep_files: + if not keep_files: os.remove(dut_pkt_file) diff --git a/tests/conftest.py b/tests/conftest.py index 76572eadd4..1d2726dc44 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -192,11 +192,12 @@ def pytest_addoption(parser): ) parser.addoption( - "--decoder_only", - help="Only run decoder parts of tests in 'codec_be_on_mr_nonselection'. Use ref encoder output bitstreams.", - action="store_true", - default=False, - ) + "--decoder_only", + help="Only run decoder parts of tests in 'codec_be_on_mr_nonselection'. Use ref encoder output bitstreams.", + action="store_true", + default=False, + ) + @pytest.fixture(scope="session", autouse=True) def update_ref(request): @@ -569,7 +570,10 @@ def dut_decoder_frontend(dut_decoder_path, request) -> DecoderFrontend: Return a :class:`conftest.DecoderFrontend` instance as DUT for the test session. """ decoder = DecoderFrontend( - dut_decoder_path, "DUT", timeout=request.config.getoption("--testcase_timeout"), fr=request.config.option.dut_fr + dut_decoder_path, + "DUT", + timeout=request.config.getoption("--testcase_timeout"), + fr=request.config.option.dut_fr, ) yield decoder -- GitLab From 86ef7ef817b232300ba9fe8cc4dacaeb06f82212 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 28 Feb 2024 12:18:56 +0100 Subject: [PATCH 20/24] use --decoder_only in 5ms/10ms comparison test --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1352eb875b..caa8267bff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -549,8 +549,8 @@ pytest-compare-20ms-and-5ms-rendering: - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2 ### run pytest - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 || exit_code5=$? - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 || exit_code10=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$? - zero_errors5=$(cat report-junit-5ms.xml | grep -c 'errors="0"') || true - zero_errors10=$(cat report-junit-10ms.xml | grep -c 'errors="0"') || true - zero_errors=1 -- GitLab From 5431b64c6ec8eb265803b9548b0955b8256435b0 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 28 Feb 2024 12:59:54 +0100 Subject: [PATCH 21/24] fix bug with netsim application --- tests/codec_be_on_mr_nonselection/test_param_file.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 0a494d7248..459964a186 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -496,14 +496,18 @@ def simulate( ): # call network simulator on REF encoder output cmd_opts[1] = f"{ref_out_dir}/{netsim_infile}" + cmd_opts[2] = f"{ref_out_dir}/{netsim_outfile}" # ref_out_file elif update_ref in [0, 2]: # call network simulator on DUT encoder output cmd_opts[1] = f"{dut_out_dir}/{netsim_infile}" + cmd_opts[2] = f"{dut_out_dir}/{netsim_outfile}" # dut_out_file if update_ref == 1 or (update_ref == 2 and not os.path.exists(ref_out_file)): cmd_opts[3] = f"{ref_out_dir}/{netsim_tracefile}" + cmd_opts[2] = f"{ref_out_dir}/{netsim_outfile}" # ref_out_file elif update_ref in [0, 2]: cmd_opts[3] = f"{dut_out_dir}/{netsim_tracefile}" + cmd_opts[2] = f"{dut_out_dir}/{netsim_outfile}" # dut_out_file run(netsim + cmd_opts, check=False) -- GitLab From f93cf20eeb5c88ac3d80a7cb10e6f8ec64c69019 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 28 Feb 2024 13:21:48 +0100 Subject: [PATCH 22/24] fix error in file removal --- tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index e088196ff1..01ac63afac 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -675,15 +675,15 @@ def sba_dec( if gain_flag != -1: short_tag_ext += f"_Gain{gain_flag}" if pca: - short_tag_ext += f"_pca" + short_tag_ext += "_pca" if SID == 1: - short_tag_ext += f"_SID_cut" + short_tag_ext += "_SID_cut" # to avoid conflicting names in case of parallel test execution, differentiate all cases long_tag_ext = "" if gain_flag != -1: long_tag_ext += f"_Gain{gain_flag}" if SID == 1: - long_tag_ext += f"_SID_cut" + long_tag_ext += "_SID_cut" dut_out_dir = f"{dut_base_path}/sba_bs/raw" ref_out_dir = f"{reference_path}/sba_bs/raw" @@ -706,11 +706,14 @@ def sba_dec( ) if update_ref == 0: + if decoder_only: + dut_in_pkt = ref_in_pkt + # call DUT decoder decoder_frontend.run( output_config, sampling_rate, - dut_in_pkt if not decoder_only else ref_in_pkt, + dut_in_pkt dut_out_raw, ) -- GitLab From cbfe430d127a91f56565824749a2421990810e48 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 28 Feb 2024 13:53:10 +0100 Subject: [PATCH 23/24] correct file removal --- tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py | 5 +++-- tests/conftest.py | 6 ++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index 01ac63afac..4f4c1a3271 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -713,7 +713,7 @@ def sba_dec( decoder_frontend.run( output_config, sampling_rate, - dut_in_pkt + dut_in_pkt, dut_out_raw, ) @@ -738,5 +738,6 @@ def sba_dec( # remove DUT output files when test result is OK (to save disk space) if not keep_files: - os.remove(dut_in_pkt) os.remove(dut_out_raw) + if not decoder_only: + os.remove(dut_in_pkt) diff --git a/tests/conftest.py b/tests/conftest.py index 1d2726dc44..bbe16f4715 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -506,10 +506,8 @@ class DecoderFrontend: try: if not os.path.exists(str(input_bitstream_path) + eid_output_suffix): result = run(eid_command, check=True) - except Exception as e: - print(result.stderr) - print(result.stdout) - pytest.fail(f"eid-xor operation failed!") + except Exception: + pytest.fail("eid-xor operation failed!") input_bitstream_path += eid_output_suffix -- GitLab From 9af5b4d83c65cdc88c94ee4bd413507e6758ac97 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 28 Feb 2024 14:39:55 +0100 Subject: [PATCH 24/24] add shorter renderer framesize tests to coverage report --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index caa8267bff..ee7c7ea915 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2040,6 +2040,8 @@ coverage-test-on-main-scheduled: - python3 tests/create_short_testvectors.py - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_ref - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_cut - lcov -c -d obj -o coverage_stv.info # extract coverage of short test vectors here -- GitLab