From b1ad63f346bdf61e2f84667c14cf1b9a7e52682d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Mon, 18 Jul 2022 14:31:46 +0200 Subject: [PATCH 01/25] cmp_custom.py: fixed usage/help print --- scripts/ivas_pytests/tests/cmp_custom.py | 2 ++ 1 file changed, 2 insertions(+) mode change 100644 => 100755 scripts/ivas_pytests/tests/cmp_custom.py diff --git a/scripts/ivas_pytests/tests/cmp_custom.py b/scripts/ivas_pytests/tests/cmp_custom.py old mode 100644 new mode 100755 index e52d6df46c..1999d7ccc5 --- a/scripts/ivas_pytests/tests/cmp_custom.py +++ b/scripts/ivas_pytests/tests/cmp_custom.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +__license__ = \ """ (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., @@ -30,6 +31,7 @@ the United Nations Convention on Contracts on the International Sales of Goods. """ +__doc__ = \ """ Script to compare samples in 2 PCM files. -- GitLab From b7b4067b0ad46bb4029716fdf732b54105fb7b1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Mon, 18 Jul 2022 14:42:27 +0200 Subject: [PATCH 02/25] setup tests folder --- {scripts/ivas_pytests/tests => tests}/cmp_custom.py | 0 {scripts/ivas_pytests => tests}/conftest.py | 0 {scripts/ivas_pytests/tests => tests}/cut_pcm.py | 0 {scripts/ivas_pytests/tests => tests}/il2mm.py | 0 {scripts/ivas_pytests/tests => tests}/requirements.txt | 0 {scripts/ivas_pytests => tests}/self_test_b.py | 0 .../test_spar_foa_bs_dec_plc.py => tests/test_sba_bs_dec_plc.py | 0 .../test_spar_foa_bs_enc.py => tests/test_sba_bs_enc.py | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename {scripts/ivas_pytests/tests => tests}/cmp_custom.py (100%) rename {scripts/ivas_pytests => tests}/conftest.py (100%) rename {scripts/ivas_pytests/tests => tests}/cut_pcm.py (100%) rename {scripts/ivas_pytests/tests => tests}/il2mm.py (100%) rename {scripts/ivas_pytests/tests => tests}/requirements.txt (100%) rename {scripts/ivas_pytests => tests}/self_test_b.py (100%) rename scripts/ivas_pytests/tests/system_tests/test_spar_foa_bs_dec_plc.py => tests/test_sba_bs_dec_plc.py (100%) rename scripts/ivas_pytests/tests/system_tests/test_spar_foa_bs_enc.py => tests/test_sba_bs_enc.py (100%) diff --git a/scripts/ivas_pytests/tests/cmp_custom.py b/tests/cmp_custom.py similarity index 100% rename from scripts/ivas_pytests/tests/cmp_custom.py rename to tests/cmp_custom.py diff --git a/scripts/ivas_pytests/conftest.py b/tests/conftest.py similarity index 100% rename from scripts/ivas_pytests/conftest.py rename to tests/conftest.py diff --git a/scripts/ivas_pytests/tests/cut_pcm.py b/tests/cut_pcm.py similarity index 100% rename from scripts/ivas_pytests/tests/cut_pcm.py rename to tests/cut_pcm.py diff --git a/scripts/ivas_pytests/tests/il2mm.py b/tests/il2mm.py similarity index 100% rename from scripts/ivas_pytests/tests/il2mm.py rename to tests/il2mm.py diff --git a/scripts/ivas_pytests/tests/requirements.txt b/tests/requirements.txt similarity index 100% rename from scripts/ivas_pytests/tests/requirements.txt rename to tests/requirements.txt diff --git a/scripts/ivas_pytests/self_test_b.py b/tests/self_test_b.py similarity index 100% rename from scripts/ivas_pytests/self_test_b.py rename to tests/self_test_b.py diff --git a/scripts/ivas_pytests/tests/system_tests/test_spar_foa_bs_dec_plc.py b/tests/test_sba_bs_dec_plc.py similarity index 100% rename from scripts/ivas_pytests/tests/system_tests/test_spar_foa_bs_dec_plc.py rename to tests/test_sba_bs_dec_plc.py diff --git a/scripts/ivas_pytests/tests/system_tests/test_spar_foa_bs_enc.py b/tests/test_sba_bs_enc.py similarity index 100% rename from scripts/ivas_pytests/tests/system_tests/test_spar_foa_bs_enc.py rename to tests/test_sba_bs_enc.py -- GitLab From 45a397696dcd77bb3c2b8e34a6ab5e7485fc85b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Mon, 18 Jul 2022 15:40:31 +0200 Subject: [PATCH 03/25] adjustments to new tests folder + renaming of spar_foa to sba --- .gitlab-ci.yml | 2 +- tests/conftest.py | 18 ++++++++-------- tests/self_test_b.py | 20 ++++++++--------- tests/test_sba_bs_dec_plc.py | 20 ++++++++--------- tests/test_sba_bs_enc.py | 42 +++++++++++++++++------------------- 5 files changed, 49 insertions(+), 53 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a66624cc9b..f60569f347 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -165,7 +165,7 @@ self-test-on-merge-request: ### run SBA pytest - exit_code=0 - - python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$? + - python3 ./tests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$? - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi - zero_errors=$(cat report-junit.xml | grep -c 'testsuite errors="0"') || true - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi diff --git a/tests/conftest.py b/tests/conftest.py index 2cc74689af..24bd512dd2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -130,9 +130,9 @@ def dut_encoder_path(request) -> Path: system = platform.system() if system == "Windows": - path = here.joinpath("../../IVAS_cod.exe") + path = here.joinpath("../IVAS_cod.exe") elif system in ["Darwin", "Linux"]: - path = here.joinpath("../../IVAS_cod") + path = here.joinpath("../IVAS_cod") else: raise ValueError(f'Wrong system "{system}"!') @@ -242,9 +242,9 @@ def ref_encoder_path(request) -> Path: system = platform.system() if system == "Windows": - path = here.joinpath("../../IVAS_cod.exe") + path = here.joinpath("../IVAS_cod.exe") elif system in ["Darwin", "Linux"]: - path = here.joinpath("../../IVAS_cod") + path = here.joinpath("../IVAS_cod") else: raise ValueError(f'Wrong system "{system}"!') @@ -264,9 +264,9 @@ def dut_decoder_path(request) -> Path: system = platform.system() if system == "Windows": - path = here.joinpath("../../IVAS_dec.exe") + path = here.joinpath("../IVAS_dec.exe") elif system in ["Darwin", "Linux"]: - path = here.joinpath("../../IVAS_dec") + path = here.joinpath("../IVAS_dec") else: raise ValueError(f'Wrong system "{system}"!') @@ -360,9 +360,9 @@ def ref_decoder_path(request) -> Path: system = platform.system() if system == "Windows": - path = here.joinpath("../../IVAS_dec.exe") + path = here.joinpath("../IVAS_dec.exe") elif system in ["Darwin", "Linux"]: - path = here.joinpath("../../IVAS_dec") + path = here.joinpath("../IVAS_dec") else: raise ValueError(f'Wrong system "{system}"!') @@ -380,7 +380,7 @@ def data_system_tests_path(request) -> Path: here = Path(__file__).parent.resolve() - path = here.joinpath("testv") + path = here.joinpath("../scripts/testv") path = str(path.resolve()) diff --git a/tests/self_test_b.py b/tests/self_test_b.py index 941739435e..917bb4c0fe 100755 --- a/tests/self_test_b.py +++ b/tests/self_test_b.py @@ -45,17 +45,17 @@ import subprocess import platform from pathlib import Path -sys.path.append('scripts/ivas_pytests/tests/') +sys.path.append('tests/') from cut_pcm import cut_samples BIN_EXT = ".exe" if platform.system() == "Windows" else "" HERE = Path(__file__).parent.resolve() -DEFAULT_ENCODER_DUT = str(HERE.joinpath(f"../../IVAS_cod{BIN_EXT}").resolve()) -DEFAULT_DECODER_DUT = str(HERE.joinpath(f"../../IVAS_dec{BIN_EXT}").resolve()) -DEFAULT_ENCODER_REF = str(HERE.joinpath(f"../../IVAS_cod_ref{BIN_EXT}").resolve()) -DEFAULT_DECODER_REF = str(HERE.joinpath(f"../../IVAS_dec_ref{BIN_EXT}").resolve()) -CREND_UNITTEST_REF = str(HERE.joinpath(f"tests/unit_tests/crend/IVAS_crend_unit_test_ref{BIN_EXT}").resolve()) -TEST_VECTOR_DIR = str(HERE.joinpath("../testv").resolve()) +DEFAULT_ENCODER_DUT = str(HERE.joinpath(f"../IVAS_cod{BIN_EXT}").resolve()) +DEFAULT_DECODER_DUT = str(HERE.joinpath(f"../IVAS_dec{BIN_EXT}").resolve()) +DEFAULT_ENCODER_REF = str(HERE.joinpath(f"../IVAS_cod_ref{BIN_EXT}").resolve()) +DEFAULT_DECODER_REF = str(HERE.joinpath(f"../IVAS_dec_ref{BIN_EXT}").resolve()) +CREND_UNITTEST_REF = str(HERE.joinpath(f"../scripts/ivas_pytests/tests/unit_tests/crend/IVAS_crend_unit_test_ref{BIN_EXT}").resolve()) +TEST_VECTOR_DIR = str(HERE.joinpath("../scripts/testv").resolve()) REFERENCE_DIR = str(HERE.joinpath("ref").resolve()) DUT_BASE_DIR = str(HERE.joinpath("dut").resolve()) @@ -105,7 +105,7 @@ def build_dut_binaries(): Build the DUT binaries. """ print("Building the DUT binaries") - dut_src_dir = str(HERE.joinpath("../..").resolve()) + dut_src_dir = str(HERE.joinpath("..").resolve()) build_enc_and_dec(dut_src_dir) build_crend_unittest(dut_src_dir) @@ -202,7 +202,7 @@ def main(argv): else: base_cmd = ["python3", "-m", "pytest"] base_cmd += [ - "scripts/ivas_pytests/tests", + "tests", "-n", args.numprocesses, "--update_ref", @@ -246,7 +246,7 @@ def main(argv): else: cmd = ["python3", "-m", "pytest"] cmd += [ - "scripts/ivas_pytests/tests", + "tests", "-n", args.numprocesses, "-v", diff --git a/tests/test_sba_bs_dec_plc.py b/tests/test_sba_bs_dec_plc.py index 7b9a2b245d..a6d6ae82f5 100644 --- a/tests/test_sba_bs_dec_plc.py +++ b/tests/test_sba_bs_dec_plc.py @@ -34,8 +34,6 @@ import shutil import errno import sys -sys.path.append('scripts/ivas_pytests/') -sys.path.append('scripts/ivas_pytests/tests/') from il2mm import il2mm from cmp_custom import cmp_custom from conftest import EncoderFrontend, DecoderFrontend @@ -62,10 +60,10 @@ def check_and_makedir(dir_path): # assumption: -# - the needed reference bitstreams are created by test_spar_foa_enc_system +# - the needed reference bitstreams are created by test_sba_enc_system # -> reference bitstreams are not any longer created as part of this test -# -> the parameters of this test (except additional parameter plc_pattern) need to be a subset of the parameters in test_spar_foa_enc_system -# -> the reference generation for this test (reference decoder output) needs to be done after completion of test_spar_foa_enc_system +# -> the parameters of this test (except additional parameter plc_pattern) need to be a subset of the parameters in test_sba_enc_system +# -> the reference generation for this test (reference decoder output) needs to be done after completion of test_sba_enc_system # -> therefore the marker create_ref_part2 @pytest.mark.create_ref_part2 @pytest.mark.parametrize("ivas_br", ivas_br_list) @@ -74,7 +72,7 @@ def check_and_makedir(dir_path): @pytest.mark.parametrize("plc_pattern", plc_patterns) @pytest.mark.parametrize("fs", sampling_rate_list) @pytest.mark.parametrize("agc", agc_list) -def test_spar_foa_plc_system( +def test_sba_plc_system( dut_decoder_frontend: DecoderFrontend, data_system_tests_path, reference_path, @@ -91,12 +89,12 @@ def test_spar_foa_plc_system( tag = tag + fs + 'c' #dec - spar_foa_dec_plc(dut_decoder_frontend, data_system_tests_path, reference_path, dut_base_path, ref_decoder_path, tag, ch_count_foa, fs, ivas_br, dtx, plc_pattern, update_ref, agc) + sba_dec_plc(dut_decoder_frontend, data_system_tests_path, reference_path, dut_base_path, ref_decoder_path, tag, ch_count_foa, fs, ivas_br, dtx, plc_pattern, update_ref, agc) ######################################################### ############ test function ############################## -def spar_foa_dec_plc( +def sba_dec_plc( decoder_frontend, test_vector_path, reference_path, @@ -119,14 +117,14 @@ def spar_foa_dec_plc( tag_out += '_AGC1' plc_tag_out = f"{tag_out}_{plc_pattern}" - dut_out_dir = f"{dut_base_path}/spar_foa_bs/raw/{plc_tag_out}" - ref_out_dir = f"{reference_path}/spar_foa_bs/raw/{plc_tag_out}" + dut_out_dir = f"{dut_base_path}/sba_bs/raw/{plc_tag_out}" + ref_out_dir = f"{reference_path}/sba_bs/raw/{plc_tag_out}" check_and_makedir(dut_out_dir) check_and_makedir(ref_out_dir) plc_file = f"{test_vector_path}/{plc_pattern}.g192" - ref_in_pkt = f"{reference_path}/spar_foa_bs/pkt/{tag_out}.pkt" + ref_in_pkt = f"{reference_path}/sba_bs/pkt/{tag_out}.pkt" if ref_decoder_path: ref_decoder = DecoderFrontend(ref_decoder_path, "REF") diff --git a/tests/test_sba_bs_enc.py b/tests/test_sba_bs_enc.py index 25bcfe7f3b..7e3f2c1601 100644 --- a/tests/test_sba_bs_enc.py +++ b/tests/test_sba_bs_enc.py @@ -39,8 +39,6 @@ import errno import shutil import sys -sys.path.append('scripts/ivas_pytests/') -sys.path.append('scripts/ivas_pytests/tests/') from il2mm import il2mm from cmp_custom import cmp_custom from cut_pcm import cut_samples @@ -108,7 +106,7 @@ def test_bypass_enc( output_config = "FOA" # enc - spar_foa_enc( + sba_enc( dut_encoder_frontend, data_system_tests_path, ref_encoder_path, @@ -126,7 +124,7 @@ def test_bypass_enc( ) # dec - spar_foa_dec( + sba_dec( dut_decoder_frontend, ref_decoder_path, reference_path, @@ -150,7 +148,7 @@ def test_bypass_enc( @pytest.mark.parametrize("tag", tag_list) @pytest.mark.parametrize("fs", sample_rate_list) @pytest.mark.parametrize("agc", agc_list) -def test_spar_foa_enc_system( +def test_sba_enc_system( dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, data_system_tests_path, @@ -176,7 +174,7 @@ def test_spar_foa_enc_system( cut_gain = "1.0" # enc - spar_foa_enc( + sba_enc( dut_encoder_frontend, data_system_tests_path, ref_encoder_path, @@ -195,7 +193,7 @@ def test_spar_foa_enc_system( ) # dec - spar_foa_dec( + sba_dec( dut_decoder_frontend, ref_decoder_path, reference_path, @@ -238,7 +236,7 @@ def test_spar_hoa2_enc_system( output_config = "HOA2" # enc - spar_foa_enc( + sba_enc( dut_encoder_frontend, data_system_tests_path, ref_encoder_path, @@ -257,7 +255,7 @@ def test_spar_hoa2_enc_system( ) # dec - spar_foa_dec( + sba_dec( dut_decoder_frontend, ref_decoder_path, reference_path, @@ -300,7 +298,7 @@ def test_spar_hoa3_enc_system( output_config = "HOA3" # enc - spar_foa_enc( + sba_enc( dut_encoder_frontend, data_system_tests_path, ref_encoder_path, @@ -319,7 +317,7 @@ def test_spar_hoa3_enc_system( ) # dec - spar_foa_dec( + sba_dec( dut_decoder_frontend, ref_decoder_path, reference_path, @@ -341,7 +339,7 @@ def test_spar_hoa3_enc_system( @pytest.mark.parametrize("dtx", dtx_set) @pytest.mark.parametrize("tag", tag_list_bw_force) @pytest.mark.parametrize("sample_rate_bw_idx", sample_rate_bw_idx_list) -def test_spar_foa_enc_BWforce_system( +def test_sba_enc_BWforce_system( dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, data_system_tests_path, @@ -364,7 +362,7 @@ def test_spar_foa_enc_BWforce_system( output_config = "FOA" # enc - spar_foa_enc( + sba_enc( dut_encoder_frontend, data_system_tests_path, ref_encoder_path, @@ -382,7 +380,7 @@ def test_spar_foa_enc_BWforce_system( ) # dec - spar_foa_dec( + sba_dec( dut_decoder_frontend, ref_decoder_path, reference_path, @@ -402,7 +400,7 @@ def test_spar_foa_enc_BWforce_system( ######################################################### ############ test function ############################## -def spar_foa_enc( +def sba_enc( encoder_frontend, test_vector_path, ref_encoder_path, @@ -422,8 +420,8 @@ def spar_foa_enc( ): ######### run cmd ##################################### - dut_out_dir = f"{dut_base_path}/spar_foa_bs/pkt" - ref_out_dir = f"{reference_path}/spar_foa_bs/pkt" + dut_out_dir = f"{dut_base_path}/sba_bs/pkt" + ref_out_dir = f"{reference_path}/sba_bs/pkt" check_and_makedir(dut_out_dir) check_and_makedir(ref_out_dir) @@ -496,7 +494,7 @@ def spar_foa_enc( ) -def spar_foa_dec( +def sba_dec( decoder_frontend, ref_decoder_path, reference_path, @@ -533,11 +531,11 @@ def spar_foa_dec( # to avoid conflicting names in case of parallel test execution, differentiate all cases long_tag_ext = f"_AGC{agc}_pca{bypass}" - dut_out_dir = f"{dut_base_path}/spar_foa_bs/raw/{tag_out}{long_tag_ext}" - ref_out_dir = f"{reference_path}/spar_foa_bs/raw/{tag_out}{short_tag_ext}" + dut_out_dir = f"{dut_base_path}/sba_bs/raw/{tag_out}{long_tag_ext}" + ref_out_dir = f"{reference_path}/sba_bs/raw/{tag_out}{short_tag_ext}" - dut_in_pkt = f"{dut_base_path}/spar_foa_bs/pkt/{tag_out}{long_tag_ext}.pkt" - ref_in_pkt = f"{reference_path}/spar_foa_bs/pkt/{tag_out}{short_tag_ext}.pkt" + dut_in_pkt = f"{dut_base_path}/sba_bs/pkt/{tag_out}{long_tag_ext}.pkt" + ref_in_pkt = f"{reference_path}/sba_bs/pkt/{tag_out}{short_tag_ext}.pkt" check_and_makedir(dut_out_dir) check_and_makedir(ref_out_dir) -- GitLab From 00337adcab19aabff2b38edc9b8f4071e1f61614 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Mon, 18 Jul 2022 16:38:37 +0200 Subject: [PATCH 04/25] added param_file test + renamed data_system_tests_path test_vector_path --- tests/conftest.py | 64 +++---- tests/self_test_b.py | 4 +- tests/test_param_file.py | 321 +++++++++++++++++++++++++++++++++++ tests/test_sba_bs_dec_plc.py | 7 +- tests/test_sba_bs_enc.py | 21 ++- 5 files changed, 370 insertions(+), 47 deletions(-) create mode 100644 tests/test_param_file.py diff --git a/tests/conftest.py b/tests/conftest.py index 24bd512dd2..6e8385e411 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -58,51 +58,57 @@ def rootdir(request): def pytest_addoption(parser): - parser.addoption("--update_ref", action="store", default="0") - parser.addoption("--p4_CL", action="store") - parser.addoption("--p4cmd_active", action="store", default="0") + parser.addoption( + "--update_ref", + action="store", + help="""Indicate whether references shall be updated. + 0: Only DUT processing, no reference generation, references need to be present. + 1: Only reference generation (unconditionally), no DUT processing. + 2: DUT processing, references are generated when not present (not supported by all tests). + """, + default="0", + ) parser.addoption( "--dut_encoder_path", action="store", - help="If specified, use given binary as DUT encoder." + help="If specified, use given binary as DUT encoder.", ) parser.addoption( "--dut_decoder_path", action="store", - help="If specified, use given binary as DUT decoder." + help="If specified, use given binary as DUT decoder.", ) parser.addoption( "--ref_encoder_path", action="store", - help="If specified, use given binary as REF encoder." + help="If specified, use given binary as REF encoder.", ) parser.addoption( "--ref_decoder_path", action="store", - help="If specified, use given binary as REF decoder." + help="If specified, use given binary as REF decoder.", ) - # TODO: rename to test_vector_path parser.addoption( - "--data_system_tests_path", + "--test_vector_path", action="store", - help="If specified, use given directory as base data directory for system tests." + help="If specified, use given directory as base directory for test vector files.", ) parser.addoption( "--reference_path", action="store", - help="If specified, use given directory as base directory for reference files." + help="If specified, use given directory as base directory for reference files.", ) parser.addoption( "--dut_base_path", action="store", - help="If specified, use given directory as base data directory for dut files." + help="If specified, use given directory as base data directory for dut files.", ) @@ -111,16 +117,6 @@ def update_ref(request): return int(request.config.getoption("--update_ref")) -@pytest.fixture(scope="session", autouse=True) -def p4_CL(request): - return request.config.option.p4_CL - - -@pytest.fixture(scope="session", autouse=True) -def p4cmd_active(request): - return int(request.config.getoption("--p4cmd_active")) - - @pytest.fixture(scope="session") def dut_encoder_path(request) -> Path: if request.config.option.dut_encoder_path: @@ -163,6 +159,7 @@ class EncoderFrontend: agc_op: Optional[int] = None, bypass_mode: Optional[int] = None, quiet_mode: Optional[bool] = True, + add_option_list: Optional[list] = None, ) -> None: command = [self._path] @@ -185,6 +182,9 @@ class EncoderFrontend: if quiet_mode: command.extend(["-q"]) + if add_option_list is not None: + command.extend(add_option_list) + # add mandatory parameters command += [ str(bitrate), @@ -233,8 +233,7 @@ def ref_encoder_path(request) -> Path: if request.config.option.ref_encoder_path: return request.config.option.ref_encoder_path - update_ref = int(request.config.getoption("--update_ref")) - if not update_ref: + if request.config.option.update_ref == "0": return None # assume default encoder when update_ref is selected, but no ref_encoder_path is specified @@ -293,6 +292,7 @@ class DecoderFrontend: output_path: Path, quiet_mode: Optional[bool] = True, plc_file: Optional[Path] = None, + add_option_list: Optional[list] = None, ) -> None: command = [self._path] @@ -303,9 +303,14 @@ class DecoderFrontend: if plc_file is not None: command.extend(["-fec", str(plc_file)]) + if add_option_list is not None: + command.extend(add_option_list) + # add mandatory parameters + # output_config is mandatory for IVAS; EVS does not have this parameter, indicated by "" + if output_config is not "": + command += [output_config] command += [ - output_config, str(output_sampling_rate), str(input_bitstream_path), str(output_path), @@ -351,8 +356,7 @@ def ref_decoder_path(request) -> Path: if request.config.option.ref_decoder_path: return request.config.option.ref_decoder_path - update_ref = int(request.config.getoption("--update_ref")) - if not update_ref: + if request.config.option.update_ref == "0": return None # assume default decoder when update_ref is selected, but no ref_decoder_path is specified @@ -374,9 +378,9 @@ def ref_decoder_path(request) -> Path: @pytest.fixture(scope="session") -def data_system_tests_path(request) -> Path: - if request.config.option.data_system_tests_path: - return request.config.option.data_system_tests_path +def test_vector_path(request) -> Path: + if request.config.option.test_vector_path: + return request.config.option.test_vector_path here = Path(__file__).parent.resolve() diff --git a/tests/self_test_b.py b/tests/self_test_b.py index 917bb4c0fe..dbebf59817 100755 --- a/tests/self_test_b.py +++ b/tests/self_test_b.py @@ -208,7 +208,7 @@ def main(argv): "--update_ref", "1", "-v", - "--data_system_tests_path", + "--test_vector_path", TEST_VECTOR_DIR, "--reference_path", REFERENCE_DIR, @@ -250,7 +250,7 @@ def main(argv): "-n", args.numprocesses, "-v", - "--data_system_tests_path", + "--test_vector_path", TEST_VECTOR_DIR, "--reference_path", REFERENCE_DIR, diff --git a/tests/test_param_file.py b/tests/test_param_file.py new file mode 100644 index 0000000000..59126bdd8d --- /dev/null +++ b/tests/test_param_file.py @@ -0,0 +1,321 @@ +""" +Execute tests specified via a parameter file. +""" + +__copyright__ = """ +(C) 2022 Baseline Development Group with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies OY, Orange, +Panasonic Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The Baseline Development Group consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies OY, Orange, +Panasonic Corporation, Qualcomm Technologies, Inc., and VoiceAge Corporation retain full ownership +rights in their respective contributions in the software. No license of any kind, including but not +limited to patent license, of any foregoing parties is hereby granted by implication, estoppel or +otherwise. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and/or fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +import os +import errno +import pytest +from cmp_custom import cmp_custom +from conftest import EncoderFrontend, DecoderFrontend + +PARAM_FILE = "scripts/config/self_test.prm" + +param_file_test_dict = {} +with open(PARAM_FILE, "r", encoding="UTF-8") as fp: + data = fp.read() + blocks = data.split("\n\n") + for block in blocks: + tag = "" + enc_opts = "" + dec_opts = "" + for line in block.split("\n"): + if line.startswith("// "): + tag = line[3:] + if line.startswith("../IVAS_cod "): + enc_opts = line[12:] + if line.startswith("../IVAS_dec "): + dec_opts = line[12:] + if tag == "" or enc_opts == "" or dec_opts == "": + # no complete parameter set + continue + if tag in param_file_test_dict: + print("non-unique tag found - ignoring new entry") + continue + param_file_test_dict[tag] = (enc_opts, dec_opts) + + +def check_and_makedir(dir_path): + if not os.path.exists(dir_path): + try: + os.makedirs(dir_path) + except OSError as e: + if e.errno != errno.EEXIST: + raise # raises the error again + + +def convert_test_string_to_tag(test_string): + """ + Convert a test string (i.e. the test tag from the parameter file) to a tag string. + Example: + in: "DFT stereo at 13.2 kbps, 16kHz in, 16kHz out, DTX on, random FEC at 5%" + out: "DFT_stereo_at_13_2_kbps_16kHz_in_16kHz_out_DTX_on_random_FEC_at_5_" + """ + # replace certain characters by "_" or remove them + tag_str = "" + replace_chars = " %.-()" + remove_chars = "," + for char in test_string: + if char in replace_chars: + tag_str += "_" + elif char not in remove_chars: + tag_str += char + # replace double underscore by single one + tag_str = "_".join(tag_str.split("__")) + return tag_str + + +@pytest.mark.create_ref +@pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys())) +def test_param_file_tests( + dut_encoder_frontend: EncoderFrontend, + dut_decoder_frontend: DecoderFrontend, + ref_encoder_path, + ref_decoder_path, + reference_path, + dut_base_path, + test_vector_path, + update_ref, + test_tag, +): + enc_opts, dec_opts = param_file_test_dict[test_tag] + + tag_str = convert_test_string_to_tag(test_tag) + + # evaluate encoder options + enc_split = enc_opts.split() + assert len(enc_split) >= 4 + + # replace "testv/" by extended test vector path + enc_split = [ + x.replace("testv", f"{test_vector_path}/param_file", 1) if x.startswith("testv/") else x + for x in enc_split + ] + + bitstream_file = enc_split.pop() + testv_file = enc_split.pop() + sampling_rate = int(enc_split.pop()) + bitrate = enc_split.pop() + + # bitrate can be a filename: remove leading "../" + if bitrate.startswith("../"): + bitrate = bitrate[3:] + + testv_base = testv_file.split("/")[-1] + if testv_base.endswith(".pcm"): + testv_base = testv_base[:-4] + + assert bitstream_file == "bit" + # in the parameter file, only "bit" is used as bitstream file name + # -> construct bitstream filename + bitstream_file = f"{testv_base}_{tag_str}.192" + + encode( + dut_encoder_frontend, + ref_encoder_path, + reference_path, + dut_base_path, + bitrate, + sampling_rate, + testv_file, + bitstream_file, + enc_split, + update_ref, + ) + + # evaluate decoder options + dec_split = dec_opts.split() + assert len(dec_split) >= 3 + + # replace "testv/" by extended test vector path + dec_split = [ + x.replace("testv", f"{test_vector_path}/param_file", 1) if x.startswith("testv/") else x + for x in dec_split + ] + # remove leading "../" + dec_split = [x[3:] if x.startswith("../") else x for x in dec_split] + + output_file = dec_split.pop() + bitstream_file_dec = dec_split.pop() + sampling_rate = int(dec_split.pop()) + if len(dec_split) > 0: + output_config = dec_split.pop() + else: + output_config = "" + + output_config_name = output_config + if "/" in output_config: + # the output config is a file + output_config_name = os.path.splitext(os.path.basename(output_config))[0] + + assert bitstream_file_dec == "bit" + # in the parameter file, only "bit" is used as bitstream file name + # -> re-use bitstream filename from encoder call + + # the output file is not the real output filename + # -> construct output filename + if output_config is not "": + output_file = f"{testv_base}_{tag_str}.dec.{output_config_name}.pcm" + else: + # EVS decoder command lines do not have an output_config: use "MONO" in the output filename + output_file = f"{testv_base}_{tag_str}.dec.MONO.pcm" + + decode( + dut_decoder_frontend, + ref_decoder_path, + reference_path, + dut_base_path, + output_config, + sampling_rate, + bitstream_file, + output_file, + dec_split, + update_ref, + ) + + # compare + if update_ref in [0, 2]: + compare( + f"{dut_base_path}/param_file/dec/{output_file}", + f"{reference_path}/param_file/dec/{output_file}", + ) + + # clean-up + # TODO: consider removing DUT output files when test result is OK (to save disk space) + + +def encode( + encoder_frontend, + ref_encoder_path, + reference_path, + dut_base_path, + bitrate, + sampling_rate, + testv_file, + bitstream_file, + enc_opts_list, + update_ref, +): + """ + Call REF and/or DUT decoder. + """ + # directories + dut_out_dir = f"{dut_base_path}/param_file/enc" + ref_out_dir = f"{reference_path}/param_file/enc" + + ref_out_file = f"{ref_out_dir}/{bitstream_file}" + dut_out_file = f"{dut_out_dir}/{bitstream_file}" + + if update_ref == 1 or update_ref == 2 and not os.path.exists(ref_out_file): + check_and_makedir(ref_out_dir) + # call REF encoder + assert ref_encoder_path + ref_encoder = EncoderFrontend(ref_encoder_path, "REF") + ref_encoder.run( + bitrate, + sampling_rate, + testv_file, + ref_out_file, + add_option_list=enc_opts_list, + ) + + if update_ref in [0, 2]: + check_and_makedir(dut_out_dir) + # call DUT encoder + encoder_frontend.run( + bitrate, + sampling_rate, + testv_file, + dut_out_file, + add_option_list=enc_opts_list, + ) + + +def decode( + decoder_frontend, + ref_decoder_path, + reference_path, + dut_base_path, + output_config, + sampling_rate, + bitstream_file, + output_file, + dec_opts_list, + update_ref, +): + """ + Call REF and/or DUT decoder. + """ + # directories + dut_out_dir = f"{dut_base_path}/param_file/dec" + ref_out_dir = f"{reference_path}/param_file/dec" + + dut_in_file = f"{dut_base_path}/param_file/enc/{bitstream_file}" + ref_in_file = f"{reference_path}/param_file/enc/{bitstream_file}" + dut_out_file = f"{dut_out_dir}/{output_file}" + ref_out_file = f"{ref_out_dir}/{output_file}" + + if update_ref == 1 or update_ref == 2 and not os.path.exists(ref_out_file): + check_and_makedir(ref_out_dir) + # call REF decoder + assert ref_decoder_path + ref_decoder = DecoderFrontend(ref_decoder_path, "REF") + ref_decoder.run( + output_config, + sampling_rate, + ref_in_file, + ref_out_file, + add_option_list=dec_opts_list, + ) + + if update_ref in [0, 2]: + check_and_makedir(dut_out_dir) + # call DUT decoder + decoder_frontend.run( + output_config, + sampling_rate, + dut_in_file, + dut_out_file, + add_option_list=dec_opts_list, + ) + + +def compare( + pcm_file_1, + pcm_file_2, +): + """ + Compare two PCM files. + Currently, both PCM files are treated like mono files. + This is just fine when checking for bit-exactness. + More advanced comparisons are possible and might come with a future update. + """ + sample_size = "2" # 16-bit samples + tolerance = "0" # zero tolerance for BE testing + cmp_result = cmp_custom(pcm_file_1, pcm_file_2, sample_size, tolerance) + assert cmp_result == 0 diff --git a/tests/test_sba_bs_dec_plc.py b/tests/test_sba_bs_dec_plc.py index a6d6ae82f5..45d21cb64f 100644 --- a/tests/test_sba_bs_dec_plc.py +++ b/tests/test_sba_bs_dec_plc.py @@ -32,11 +32,10 @@ import os import pytest import shutil import errno -import sys from il2mm import il2mm from cmp_custom import cmp_custom -from conftest import EncoderFrontend, DecoderFrontend +from conftest import DecoderFrontend #params tag_list = ['stvFOA'] @@ -74,7 +73,7 @@ def check_and_makedir(dir_path): @pytest.mark.parametrize("agc", agc_list) def test_sba_plc_system( dut_decoder_frontend: DecoderFrontend, - data_system_tests_path, + test_vector_path, reference_path, dut_base_path, ref_decoder_path, @@ -89,7 +88,7 @@ def test_sba_plc_system( tag = tag + fs + 'c' #dec - sba_dec_plc(dut_decoder_frontend, data_system_tests_path, reference_path, dut_base_path, ref_decoder_path, tag, ch_count_foa, fs, ivas_br, dtx, plc_pattern, update_ref, agc) + sba_dec_plc(dut_decoder_frontend, test_vector_path, reference_path, dut_base_path, ref_decoder_path, tag, ch_count_foa, fs, ivas_br, dtx, plc_pattern, update_ref, agc) ######################################################### diff --git a/tests/test_sba_bs_enc.py b/tests/test_sba_bs_enc.py index 7e3f2c1601..8fc8db12ad 100644 --- a/tests/test_sba_bs_enc.py +++ b/tests/test_sba_bs_enc.py @@ -37,7 +37,6 @@ import os import pytest import errno import shutil -import sys from il2mm import il2mm from cmp_custom import cmp_custom @@ -84,7 +83,7 @@ def check_and_makedir(dir_path): def test_bypass_enc( dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, - data_system_tests_path, + test_vector_path, reference_path, dut_base_path, ref_encoder_path, @@ -108,7 +107,7 @@ def test_bypass_enc( # enc sba_enc( dut_encoder_frontend, - data_system_tests_path, + test_vector_path, ref_encoder_path, reference_path, dut_base_path, @@ -151,7 +150,7 @@ def test_bypass_enc( def test_sba_enc_system( dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, - data_system_tests_path, + test_vector_path, reference_path, dut_base_path, ref_encoder_path, @@ -176,7 +175,7 @@ def test_sba_enc_system( # enc sba_enc( dut_encoder_frontend, - data_system_tests_path, + test_vector_path, ref_encoder_path, reference_path, dut_base_path, @@ -216,7 +215,7 @@ def test_sba_enc_system( def test_spar_hoa2_enc_system( dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, - data_system_tests_path, + test_vector_path, reference_path, dut_base_path, ref_encoder_path, @@ -238,7 +237,7 @@ def test_spar_hoa2_enc_system( # enc sba_enc( dut_encoder_frontend, - data_system_tests_path, + test_vector_path, ref_encoder_path, reference_path, dut_base_path, @@ -278,7 +277,7 @@ def test_spar_hoa2_enc_system( def test_spar_hoa3_enc_system( dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, - data_system_tests_path, + test_vector_path, reference_path, dut_base_path, ref_encoder_path, @@ -300,7 +299,7 @@ def test_spar_hoa3_enc_system( # enc sba_enc( dut_encoder_frontend, - data_system_tests_path, + test_vector_path, ref_encoder_path, reference_path, dut_base_path, @@ -342,7 +341,7 @@ def test_spar_hoa3_enc_system( def test_sba_enc_BWforce_system( dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, - data_system_tests_path, + test_vector_path, reference_path, dut_base_path, ref_encoder_path, @@ -364,7 +363,7 @@ def test_sba_enc_BWforce_system( # enc sba_enc( dut_encoder_frontend, - data_system_tests_path, + test_vector_path, ref_encoder_path, reference_path, dut_base_path, -- GitLab From c8c38d0d5a3c795ea4bb834dc0ba7ffb48fdc228 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Mon, 18 Jul 2022 17:04:22 +0200 Subject: [PATCH 05/25] fix: use test_vector_path without param_file extension --- tests/test_param_file.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_param_file.py b/tests/test_param_file.py index 59126bdd8d..a068243d0d 100644 --- a/tests/test_param_file.py +++ b/tests/test_param_file.py @@ -111,9 +111,9 @@ def test_param_file_tests( enc_split = enc_opts.split() assert len(enc_split) >= 4 - # replace "testv/" by extended test vector path + # replace "testv/" by test vector path enc_split = [ - x.replace("testv", f"{test_vector_path}/param_file", 1) if x.startswith("testv/") else x + x.replace("testv", f"{test_vector_path}", 1) if x.startswith("testv/") else x for x in enc_split ] @@ -152,9 +152,9 @@ def test_param_file_tests( dec_split = dec_opts.split() assert len(dec_split) >= 3 - # replace "testv/" by extended test vector path + # replace "testv/" by test vector path dec_split = [ - x.replace("testv", f"{test_vector_path}/param_file", 1) if x.startswith("testv/") else x + x.replace("testv", f"{test_vector_path}", 1) if x.startswith("testv/") else x for x in dec_split ] # remove leading "../" -- GitLab From a1fb2fb0507c3e36cca1832f25e8656d12f511f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Fri, 29 Jul 2022 16:44:43 +0200 Subject: [PATCH 06/25] added docstring to custom fixtures some fixtures now fail with pytest.exit() adjusted default names of reference binaries --- tests/conftest.py | 78 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 17 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 6e8385e411..7e06c43db7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -54,6 +54,9 @@ def log_dbg_msg(message): @pytest.fixture(scope="session", autouse=True) def rootdir(request): + """ + Return root directory for tests. + """ return str(request.config.rootdir) @@ -114,11 +117,20 @@ def pytest_addoption(parser): @pytest.fixture(scope="session", autouse=True) def update_ref(request): + """ + Return indication whether references shall be updated. + 0: Only DUT processing, no reference generation. + 1: Only reference generation (unconditionally), no DUT processing. + 2: DUT processing, references are generated when not present. + """ return int(request.config.getoption("--update_ref")) @pytest.fixture(scope="session") -def dut_encoder_path(request) -> Path: +def dut_encoder_path(request) -> str: + """ + Return path of DUT encoder binary. + """ if request.config.option.dut_encoder_path: return request.config.option.dut_encoder_path @@ -134,7 +146,8 @@ def dut_encoder_path(request) -> Path: path = str(path.resolve()) - assert os.path.isfile(path) + if not os.path.isfile(path): + pytest.exit(f"\nDUT encoder binary {path} not found!\n!") return path @@ -221,6 +234,9 @@ class EncoderFrontend: @pytest.fixture(scope="function") def dut_encoder_frontend(dut_encoder_path) -> EncoderFrontend: + """ + Return a :class:`conftest.EncoderFrontend` instance as DUT for the test session. + """ encoder = EncoderFrontend(dut_encoder_path, "DUT") yield encoder @@ -229,33 +245,40 @@ def dut_encoder_frontend(dut_encoder_path) -> EncoderFrontend: @pytest.fixture(scope="session") -def ref_encoder_path(request) -> Path: +def ref_encoder_path(request) -> str: + """ + Return path of REF encoder binary. + """ if request.config.option.ref_encoder_path: return request.config.option.ref_encoder_path if request.config.option.update_ref == "0": return None - # assume default encoder when update_ref is selected, but no ref_encoder_path is specified + # assume specifically named encoder when update_ref is selected, but no ref_encoder_path is specified here = Path(__file__).parent.resolve() system = platform.system() if system == "Windows": - path = here.joinpath("../IVAS_cod.exe") + path = here.joinpath("../IVAS_cod_ref.exe") elif system in ["Darwin", "Linux"]: - path = here.joinpath("../IVAS_cod") + path = here.joinpath("../IVAS_cod_ref") else: raise ValueError(f'Wrong system "{system}"!') path = str(path.resolve()) - assert os.path.isfile(path) + if not os.path.isfile(path): + pytest.exit(f"\nREF encoder binary {path} not found!\n!") return path @pytest.fixture(scope="session") -def dut_decoder_path(request) -> Path: +def dut_decoder_path(request) -> str: + """ + Return path of DUT decoder binary. + """ if request.config.option.dut_decoder_path: return request.config.option.dut_decoder_path @@ -271,7 +294,8 @@ def dut_decoder_path(request) -> Path: path = str(path.resolve()) - assert os.path.isfile(path) + if not os.path.isfile(path): + pytest.exit(f"\nDUT decoder binary {path} not found!\n!") return path @@ -344,6 +368,9 @@ class DecoderFrontend: @pytest.fixture(scope="function") def dut_decoder_frontend(dut_decoder_path) -> DecoderFrontend: + """ + Return a :class:`conftest.DecoderFrontend` instance as DUT for the test session. + """ decoder = DecoderFrontend(dut_decoder_path, "DUT") yield decoder @@ -352,33 +379,40 @@ def dut_decoder_frontend(dut_decoder_path) -> DecoderFrontend: @pytest.fixture(scope="session") -def ref_decoder_path(request) -> Path: +def ref_decoder_path(request) -> str: + """ + Return path of REF decoder binary. + """ if request.config.option.ref_decoder_path: return request.config.option.ref_decoder_path if request.config.option.update_ref == "0": return None - # assume default decoder when update_ref is selected, but no ref_decoder_path is specified + # assume specifically named decoder when update_ref is selected, but no ref_decoder_path is specified here = Path(__file__).parent.resolve() system = platform.system() if system == "Windows": - path = here.joinpath("../IVAS_dec.exe") + path = here.joinpath("../IVAS_dec_ref.exe") elif system in ["Darwin", "Linux"]: - path = here.joinpath("../IVAS_dec") + path = here.joinpath("../IVAS_dec_ref") else: raise ValueError(f'Wrong system "{system}"!') path = str(path.resolve()) - assert os.path.isfile(path) + if not os.path.isfile(path): + pytest.exit(f"\nREF decoder binary {path} not found!\n!") return path @pytest.fixture(scope="session") -def test_vector_path(request) -> Path: +def test_vector_path(request) -> str: + """ + Return base directory of test vector files. + """ if request.config.option.test_vector_path: return request.config.option.test_vector_path @@ -392,7 +426,10 @@ def test_vector_path(request) -> Path: @pytest.fixture(scope="session") -def reference_path(request) -> Path: +def reference_path(request) -> str: + """ + Return base directory of reference files. + """ if request.config.option.reference_path: return request.config.option.reference_path @@ -402,11 +439,18 @@ def reference_path(request) -> Path: path = str(path.resolve()) + if request.config.option.update_ref == "0": + if not os.path.isdir(path): + pytest.exit(f"\nREF path {path} not found!\nPlease generate the references, first!\n!") + return path @pytest.fixture(scope="session") -def dut_base_path(request) -> Path: +def dut_base_path(request) -> str: + """ + Return base data directory for dut files. + """ if request.config.option.dut_base_path: return request.config.option.dut_base_path -- GitLab From 378484d4534c42fae4003afae3669043440391fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Wed, 3 Aug 2022 17:19:15 +0200 Subject: [PATCH 07/25] added new script create_short_testvectors.py --- tests/create_short_testvectors.py | 55 +++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100755 tests/create_short_testvectors.py diff --git a/tests/create_short_testvectors.py b/tests/create_short_testvectors.py new file mode 100755 index 0000000000..a5f50c6969 --- /dev/null +++ b/tests/create_short_testvectors.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +__license__ = \ +""" + (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository. All Rights Reserved. + + This software is protected by copyright law and by international treaties. + The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository retain full ownership rights in their respective contributions in + the software. This notice grants no license of any kind, including but not limited to patent + license, nor is any license granted by implication, estoppel or otherwise. + + Contributors are required to enter into the IVAS codec Public Collaboration agreement before making + contributions. + + This software is provided "AS IS", without any express or implied warranties. The software is in the + development stage. It is intended exclusively for experts who have experience with such software and + solely for the purpose of inspection. All implied warranties of non-infringement, merchantability + and fitness for a particular purpose are hereby disclaimed and excluded. + + Any dispute, controversy or claim arising under or in relation to providing this software shall be + submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in + accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and + the United Nations Convention on Contracts on the International Sales of Goods. +""" + +__doc__ = \ +""" +Create short (5sec) testvectors. +""" + +from pathlib import Path +from cut_pcm import cut_samples + +HERE = Path(__file__).parent.resolve() +TEST_VECTOR_DIR = str(HERE.joinpath("../scripts/testv").resolve()) + +NUM_CHANNELS = "4" # currently only FOA +CUT_FROM = "0.0" +CUT_LEN = "5.0" +for fs in ['48', '32', '16']: + in_file = f"{TEST_VECTOR_DIR}/stvFOA{fs}c.pcm" + cut_gain = "1.0" + cut_file = f"{TEST_VECTOR_DIR}/stvFOA{fs}c_cut.pcm" + cut_samples(in_file, cut_file, NUM_CHANNELS, fs + "000", CUT_FROM, CUT_LEN, cut_gain) + cut_gain = "16.0" + cut_file = f"{TEST_VECTOR_DIR}/stvFOA{fs}c_cut_{cut_gain}.pcm" + cut_samples(in_file, cut_file, NUM_CHANNELS, fs + "000", CUT_FROM, CUT_LEN, cut_gain) -- GitLab From 5178593da5ee6d9d35dfd6e467e6d4c8d55a3d24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Wed, 3 Aug 2022 17:24:32 +0200 Subject: [PATCH 08/25] added README-md to tests folder --- tests/README.md | 107 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 tests/README.md diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000000..dc22492fae --- /dev/null +++ b/tests/README.md @@ -0,0 +1,107 @@ +# IVAS tests + +The IVAS tests are using the [pytest](https://docs.pytest.org/) framework. + +## Installing test dependencies + +To use the `pytest` framework, you will need a few Python packages (in addition to Python itself). +As with other Python packages, there are different possibilities to install those packages. +Please chose the option that works best for you. + +`Note`: +The installation of Python is not described, here. +In the following, it is assumed that `Python >= 3.7` is already installed / present. + +### Global install + +```bash +pip install -r tests/requirements.txt +``` + +### User install + +```bash +pip install --user -r tests/requirements.txt +``` + +### Virtual environment install + +```bash +# set up virtual environment +python3 -m venv VENV_NAME +# change to virtual environment +source VENV_NAME/bin/activate +# install required packages +pip install -r tests/requirements.txt +``` + +## Preparing the tests + +`Note:` +Currently, shortened test vectors are used to speed up the testing. +Those shortened test vectors, some with gain adjustment, need to be created, once. + +```bash +# create shortened test vectors +python3 tests/create_short_testvectors.py +``` + +The tests rely on references which need to be generated upfront using reference binaries. +When the reference binaries are named `IVAS_cod_ref(.exe)` and `IVAS_dec_ref.(exe)`, pytest will find and use them. +When the reference binaries are named differently, you need to specify them via the `--ref_encoder_path` and `--ref_decoder_path` options. + +The tests will used the binaries `IVAS_cod(.exe)` and `IVAS_dec.(exe)` for testing. Please make sure that the binaries have been built before running the tests. + +```bash +# create references +# the following binaies need to be present: +# - IVAS_cod(.exe) +# - IVAS_dec.(exe) +# - IVAS_cod_ref(.exe) +# - IVAS_dec_ref.(exe) +# pytest command lines to be executed from project root folder: +pytest tests -n auto --update_ref 1 -m create_ref +pytest tests -n auto --update_ref 1 -m create_ref_part2 +``` + +## Running the tests + +To run all tests from the tests folder: + +```bash +# pytest command line to be executed from project root folder: +pytest tests -n auto +``` + +## Re-running some tests + +When there are test failures, you may want to run, after having fixed the code, only those test cases which had failures. This can be achieved using the `--last-failed` option. + +```bash +# rerun only the tests that failed at the last run +pytest tests -n auto --last-failed +``` + +To run a specific test case, you can e.g. pick a test case from the `short test summary info` and use that test case as an argument to `pytest`. E.g. + +```bash +# run a specific test case +pytest tests/test_sba_bs_dec_plc.py::test_sba_plc_system[0-48-PLperc12mblen5-stvFOA-0-32000] +``` + +More ways to select which tests to run: + +```bash +# run all tests within a module +pytest tests/test_sba_bs_dec_plc.py +# run a specific test from a module +pytest tests/test_sba_bs_dec_plc.py::test_sba_plc_system +``` + +## Some pytest hints + +When there a many test failures, you can use the `-x` (or `--exitfirst`) option to stop testing on the first failure. + +Commonly used options like `-n auto` can be added to addopts within the [pytest] section in `pytest.ini`. This saves some typing when calling `pytest`. + +The `-v` (or `--verbose`) option is usually helpful to see what is going on. Therefore, `-v` is currently part of addopts in `pytest.ini`. If you don't like this verbosity, you can specify the `-q` (`--quiet`) option when running `pytest`. -- GitLab From b6368612c049ef3700be39e1e18dc5568851b3e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Wed, 3 Aug 2022 18:20:04 +0200 Subject: [PATCH 09/25] - removed call to self_test.py - now covered by pytest (test_param_file) - removed call to self_test_b.py - pytest is now called directly --- .gitlab-ci.yml | 49 +++++++++++++++++++------------------------------ 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f60569f347..82086894ec 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,7 @@ variables: TESTV_DIR: "/usr/local/testv" + EXIT_CODE_NON_BE: 123 + EXIT_CODE_FAIL: 1 # prevent running two pipelines on pushes to merge request branches @@ -98,7 +100,7 @@ codec-smoke-test: # compare bit exactness between target and source branch -self-test-on-merge-request: +pytest-on-merge-request: extends: .test-job-linux stage: compare needs: [ "build-codec-linux-cmake", "codec-smoke-test" ] @@ -138,48 +140,35 @@ self-test-on-merge-request: - mv IVAS_dec ../IVAS_dec_ref - cd .. - ### re-checkout the commit from the source branch to have up-to-date self_test.py and scripts/testv (and actually everything) + ### re-checkout the commit from the source branch to have up-to-date test scripts and test vectors (and actually everything) - git checkout $source_branch_commit_sha - ### run selftest - - ls -altr scripts/testv - - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test | tee test_output.txt - - ### analyse test output - - # some helper variables - "|| true" to prevent failures from grep not finding anything + # helper variable - "|| true" to prevent failures from grep not finding anything - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true - - run_errors=$(cat test_output.txt | grep -c "test conditions had run errors") || true - - bitexact=$(cat test_output.txt | grep -c "All [0-9]* tests are bitexact") || true - - EXIT_CODE_NON_BE=123 - - EXIT_CODE_FAIL=1 - - - selftest_exit_code=0 - - # check for crashes during the test, if any happened, fail the test - - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; exit $EXIT_CODE_FAIL; fi - - # check for non bitexact output and store exit code to also always run the SBA pytest - - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_FAIL; fi - - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_NON_BE; fi - ### run SBA pytest + ### prepare pytest + # create short test vectors + - python3 tests/create_short_testvectors.py + # rename test binaries back + - mv IVAS_cod_test IVAS_cod + - mv IVAS_dec_test IVAS_dec + # create references + - python3 -m pytest tests -n auto --update_ref 1 -m create_ref + - python3 -m pytest tests -n auto --update_ref 1 -m create_ref_part2 + + ### run pytest - exit_code=0 - - python3 ./tests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$? + - python3 -m pytest tests -n auto --junit-xml=report-junit.xml || exit_code=$? - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi - zero_errors=$(cat report-junit.xml | grep -c 'testsuite errors="0"') || true - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi; - # return exit code from selftest if everything went well with the pytest run - - exit $selftest_exit_code allow_failure: exit_codes: - - 123 + - $EXIT_CODE_NON_BE artifacts: paths: - - test_output.txt - - scripts/test/logs - - scripts/ref/logs + - report-junit.xml reports: junit: report-junit.xml -- GitLab From 43127df7869c6119bb79bc5996dffeba6230fe21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Wed, 3 Aug 2022 18:35:41 +0200 Subject: [PATCH 10/25] fix invalid yaml: variables are strings, but integer is needed for allow_failure exit codes --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 82086894ec..bdccb2fce3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -165,7 +165,7 @@ pytest-on-merge-request: - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi; allow_failure: exit_codes: - - $EXIT_CODE_NON_BE + - 123 artifacts: paths: - report-junit.xml -- GitLab From a770935cd17c604e5e7af3ecde26d2e844823415 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Wed, 3 Aug 2022 19:35:58 +0200 Subject: [PATCH 11/25] fix invalid yaml: not all merge conflicts were not properly resolved --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1c25df6ed9..3ae03800c3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -238,6 +238,7 @@ asan-on-merge-request-linux: # compare bit exactness between target and source branch pytest-on-merge-request: + extends: - .test-job-linux - .rules-merge-request stage: compare -- GitLab From e70e8f0d7455e454ed286d74ecb9b1f17d48ff7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Mon, 19 Sep 2022 17:55:11 +0200 Subject: [PATCH 12/25] pytest: added option --param_file to specify .prm file --- tests/conftest.py | 12 ++++++++++-- tests/test_param_file.py | 4 ++-- tests/testconfig.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 tests/testconfig.py diff --git a/tests/conftest.py b/tests/conftest.py index 7e06c43db7..c49b6d3d83 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -35,7 +35,7 @@ from subprocess import run import textwrap from typing import Optional import os - +import testconfig import pytest logger = logging.getLogger(__name__) @@ -114,6 +114,12 @@ def pytest_addoption(parser): help="If specified, use given directory as base data directory for dut files.", ) + parser.addoption( + "--param_file", + action="store", + help="If specified, use given param file in test_param_file.", + ) + @pytest.fixture(scope="session", autouse=True) def update_ref(request): @@ -332,7 +338,7 @@ class DecoderFrontend: # add mandatory parameters # output_config is mandatory for IVAS; EVS does not have this parameter, indicated by "" - if output_config is not "": + if output_config != "": command += [output_config] command += [ str(output_sampling_rate), @@ -473,3 +479,5 @@ def pytest_configure(config): config.addinivalue_line( "markers", "create_ref_part2: reference creation test that depends on create_ref references" ) + if config.option.param_file: + testconfig.PARAM_FILE = config.option.param_file diff --git a/tests/test_param_file.py b/tests/test_param_file.py index a068243d0d..be4ffe62ec 100644 --- a/tests/test_param_file.py +++ b/tests/test_param_file.py @@ -33,8 +33,8 @@ import errno import pytest from cmp_custom import cmp_custom from conftest import EncoderFrontend, DecoderFrontend +from testconfig import PARAM_FILE -PARAM_FILE = "scripts/config/self_test.prm" param_file_test_dict = {} with open(PARAM_FILE, "r", encoding="UTF-8") as fp: @@ -179,7 +179,7 @@ def test_param_file_tests( # the output file is not the real output filename # -> construct output filename - if output_config is not "": + if output_config != "": output_file = f"{testv_base}_{tag_str}.dec.{output_config_name}.pcm" else: # EVS decoder command lines do not have an output_config: use "MONO" in the output filename diff --git a/tests/testconfig.py b/tests/testconfig.py new file mode 100644 index 0000000000..9837764910 --- /dev/null +++ b/tests/testconfig.py @@ -0,0 +1,35 @@ +__license__ = """ +(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +__doc__ = """ +To configure test modules. +""" + +PARAM_FILE = "scripts/config/self_test.prm" -- GitLab From 83926636c0e4de9b2ba70c2bea8e1d302f4b193c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Mon, 19 Sep 2022 18:01:20 +0200 Subject: [PATCH 13/25] pytest: added new script rub_pytests.py - replaces self_test_b.py --- tests/create_short_testvectors.py | 24 ++- tests/run_pytests.py | 128 ++++++++++++++ tests/self_test_b.py | 273 ------------------------------ 3 files changed, 144 insertions(+), 281 deletions(-) create mode 100755 tests/run_pytests.py delete mode 100755 tests/self_test_b.py diff --git a/tests/create_short_testvectors.py b/tests/create_short_testvectors.py index a5f50c6969..8cff1566b3 100755 --- a/tests/create_short_testvectors.py +++ b/tests/create_short_testvectors.py @@ -36,6 +36,7 @@ __doc__ = \ Create short (5sec) testvectors. """ +import sys from pathlib import Path from cut_pcm import cut_samples @@ -45,11 +46,18 @@ TEST_VECTOR_DIR = str(HERE.joinpath("../scripts/testv").resolve()) NUM_CHANNELS = "4" # currently only FOA CUT_FROM = "0.0" CUT_LEN = "5.0" -for fs in ['48', '32', '16']: - in_file = f"{TEST_VECTOR_DIR}/stvFOA{fs}c.pcm" - cut_gain = "1.0" - cut_file = f"{TEST_VECTOR_DIR}/stvFOA{fs}c_cut.pcm" - cut_samples(in_file, cut_file, NUM_CHANNELS, fs + "000", CUT_FROM, CUT_LEN, cut_gain) - cut_gain = "16.0" - cut_file = f"{TEST_VECTOR_DIR}/stvFOA{fs}c_cut_{cut_gain}.pcm" - cut_samples(in_file, cut_file, NUM_CHANNELS, fs + "000", CUT_FROM, CUT_LEN, cut_gain) + + +def create_short_testvectors(): + for fs in ['48', '32', '16']: + in_file = f"{TEST_VECTOR_DIR}/stvFOA{fs}c.pcm" + cut_gain = "1.0" + cut_file = f"{TEST_VECTOR_DIR}/stvFOA{fs}c_cut.pcm" + cut_samples(in_file, cut_file, NUM_CHANNELS, fs + "000", CUT_FROM, CUT_LEN, cut_gain) + cut_gain = "16.0" + cut_file = f"{TEST_VECTOR_DIR}/stvFOA{fs}c_cut_{cut_gain}.pcm" + cut_samples(in_file, cut_file, NUM_CHANNELS, fs + "000", CUT_FROM, CUT_LEN, cut_gain) + + +if __name__ == "__main__": + sys.exit(create_short_testvectors()) diff --git a/tests/run_pytests.py b/tests/run_pytests.py new file mode 100755 index 0000000000..cd94d27a1f --- /dev/null +++ b/tests/run_pytests.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 + +""" + (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository. All Rights Reserved. + + This software is protected by copyright law and by international treaties. + The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository retain full ownership rights in their respective contributions in + the software. This notice grants no license of any kind, including but not limited to patent + license, nor is any license granted by implication, estoppel or otherwise. + + Contributors are required to enter into the IVAS codec Public Collaboration agreement before making + contributions. + + This software is provided "AS IS", without any express or implied warranties. The software is in the + development stage. It is intended exclusively for experts who have experience with such software and + solely for the purpose of inspection. All implied warranties of non-infringement, merchantability + and fitness for a particular purpose are hereby disclaimed and excluded. + + Any dispute, controversy or claim arising under or in relation to providing this software shall be + submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in + accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and + the United Nations Convention on Contracts on the International Sales of Goods. +""" + +""" +Script to run the pytest tests. + +Test prerequisites are checked for and check failures are reported. +When prerequisites are met, the pytest test is executed. +""" + +import os +import sys +import argparse +import subprocess +import platform +from pathlib import Path + +sys.path.append('tests/') +from create_short_testvectors import create_short_testvectors + +BIN_EXT = ".exe" if platform.system() == "Windows" else "" +HERE = Path(__file__).parent.resolve() +DEFAULT_ENCODER_DUT = str(HERE.joinpath(f"../IVAS_cod{BIN_EXT}").resolve()) +DEFAULT_DECODER_DUT = str(HERE.joinpath(f"../IVAS_dec{BIN_EXT}").resolve()) +DEFAULT_ENCODER_REF = str(HERE.joinpath(f"../IVAS_cod_ref{BIN_EXT}").resolve()) +DEFAULT_DECODER_REF = str(HERE.joinpath(f"../IVAS_dec_ref{BIN_EXT}").resolve()) +REFERENCE_DIR = str(HERE.joinpath("ref").resolve()) + + +def main(argv): + # check for python >= 3.7 + if sys.version_info[0] < 3 or sys.version_info[1] < 7: + sys.exit("This script is written for Python >= 3.7. Found: " + platform.python_version()) + + parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + "--create_only", + action="store_true", + default=False, + help="Create references when needed, but don't run the tests" + ) + parser.add_argument( + "--numprocesses", + action="store", + default="auto", + help="Number of processes to use in pytest (default: auto)", + ) + + args = parser.parse_args(argv[1:]) + + # check for DUT binaries + if not os.path.exists(DEFAULT_ENCODER_DUT) or not os.path.exists(DEFAULT_DECODER_DUT): + sys.exit(f"Need DUT binaries {DEFAULT_ENCODER_DUT} and {DEFAULT_DECODER_DUT}. Please create the binaries.") + + # check for references + if os.path.exists(REFERENCE_DIR): + print(f"Using existing references directory {REFERENCE_DIR}") + else: + # check for REF binaries + print(f"References directory {REFERENCE_DIR} does not exist.") + if not os.path.exists(DEFAULT_ENCODER_REF) or not os.path.exists(DEFAULT_DECODER_REF): + sys.exit(f"Need REF binaries {DEFAULT_ENCODER_REF} and {DEFAULT_DECODER_REF}. Please create the binaries.") + + # create references + print(f"Creating references within the references directory {REFERENCE_DIR}") + create_short_testvectors() + if platform.system() == "Windows": + base_cmd = ["pytest"] + else: + base_cmd = ["python3", "-m", "pytest"] + base_cmd += [ + "tests", + "-n", + args.numprocesses, + "--update_ref", + "1", + ] + subprocess.run(base_cmd + ["-m", "create_ref"], check=False) + subprocess.run(base_cmd + ["-m", "create_ref_part2"], check=False) + + if args.create_only: + return + + # run pytest + if platform.system() == "Windows": + cmd = ["pytest"] + else: + cmd = ["python3", "-m", "pytest"] + cmd += [ + "tests", + "-n", + args.numprocesses, + ] + result = subprocess.run(cmd, check=False) + return result.returncode + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tests/self_test_b.py b/tests/self_test_b.py deleted file mode 100755 index dbebf59817..0000000000 --- a/tests/self_test_b.py +++ /dev/null @@ -1,273 +0,0 @@ -#!/usr/bin/env python3 - -""" - (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. -""" - -""" -Script to run the pytest tests. - -Step 1: Set the stage for the pytest run. - -Step 2: Run pytest. -""" - -import os -import sys -import argparse -import subprocess -import platform -from pathlib import Path - -sys.path.append('tests/') -from cut_pcm import cut_samples - -BIN_EXT = ".exe" if platform.system() == "Windows" else "" -HERE = Path(__file__).parent.resolve() -DEFAULT_ENCODER_DUT = str(HERE.joinpath(f"../IVAS_cod{BIN_EXT}").resolve()) -DEFAULT_DECODER_DUT = str(HERE.joinpath(f"../IVAS_dec{BIN_EXT}").resolve()) -DEFAULT_ENCODER_REF = str(HERE.joinpath(f"../IVAS_cod_ref{BIN_EXT}").resolve()) -DEFAULT_DECODER_REF = str(HERE.joinpath(f"../IVAS_dec_ref{BIN_EXT}").resolve()) -CREND_UNITTEST_REF = str(HERE.joinpath(f"../scripts/ivas_pytests/tests/unit_tests/crend/IVAS_crend_unit_test_ref{BIN_EXT}").resolve()) -TEST_VECTOR_DIR = str(HERE.joinpath("../scripts/testv").resolve()) -REFERENCE_DIR = str(HERE.joinpath("ref").resolve()) -DUT_BASE_DIR = str(HERE.joinpath("dut").resolve()) - - -def build_enc_and_dec(src_dir): - """ - Build the encoder and decoder binaries. - """ - if platform.system() == "Windows": - olddir = os.getcwd() - os.chdir(src_dir) - os.chdir("Workspace_msvc") - command = ["MSBuild.exe", "Workspace_msvc.sln", "/t:Clean", "/p:configuration=Release", "/p:Platform=Win32"] - subprocess.run(command, check=True) - command = ["MSBuild.exe", "Workspace_msvc.sln", "/property:configuration=Release", "/p:Platform=Win32"] - subprocess.run(command, check=True) - os.chdir(olddir) - else: - command = ["make", "-C", src_dir, "clean"] - subprocess.run(command, check=True) - command = ["make", "-C", src_dir] - subprocess.run(command, check=True) - - -def build_crend_unittest(src_dir): - """ - Build the crend unit test binary. - """ - crend_dir = f"{src_dir}/scripts/ivas_pytests/tests/unit_tests/crend" - if platform.system() == "Windows": - olddir = os.getcwd() - os.chdir(crend_dir) - # command = ["MSBuild.exe", "ivas_crend_unit_test.sln", "/t:Clean", "/p:configuration=Release", "/p:Platform=Win32"] - # subprocess.run(command, check=True) - command = ["MSBuild.exe", "ivas_crend_unit_test.sln", "/property:configuration=Release", "/p:Platform=Win32"] - subprocess.run(command, check=True) - os.chdir(olddir) - else: - # command = ["make", "-C", src_dir, "clean"] - # subprocess.run(command, check=True) - command = ["make", "-C", src_dir, "IVAS_crend_unit_test"] - subprocess.run(command, check=True) - - -def build_dut_binaries(): - """ - Build the DUT binaries. - """ - print("Building the DUT binaries") - dut_src_dir = str(HERE.joinpath("..").resolve()) - build_enc_and_dec(dut_src_dir) - build_crend_unittest(dut_src_dir) - - -def create_short_testvectors(): - """ - Create short (5sec) testvectors. - """ - print("Creating short (5sec) testvectors") - num_channels = "4" # currently only FOA - cut_from = "0.0" - cut_len = "5.0" - for fs in ['48', '32', '16']: - in_file = f"{TEST_VECTOR_DIR}/stvFOA{fs}c.pcm" - cut_gain = "1.0" - cut_file = f"{TEST_VECTOR_DIR}/stvFOA{fs}c_cut.pcm" - cut_samples(in_file, cut_file, num_channels, fs + "000", cut_from, cut_len, cut_gain) - cut_gain = "16.0" - cut_file = f"{TEST_VECTOR_DIR}/stvFOA{fs}c_cut_{cut_gain}.pcm" - cut_samples(in_file, cut_file, num_channels, fs + "000", cut_from, cut_len, cut_gain) - - -def main(argv): - # check for python >= 3.7 - if sys.version_info[0] < 3 or sys.version_info[1] < 7: - sys.exit("This script is written for Python >= 3.7. Found: " + platform.python_version()) - - parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument( - "--create_only", - action="store_true", - default=False, - help="Create references when needed, but don't run the tests" - ) - parser.add_argument( - "--numprocesses", - action="store", - default="auto", - help="Number of processes to use in pytest (default: auto)", - ) - parser.add_argument("--encref", help=f"REF encoder binary (default:{DEFAULT_ENCODER_REF})") - parser.add_argument("--decref", help=f"REF decoder binary (default:{DEFAULT_DECODER_REF})") - parser.add_argument("--encdut", help=f"DUT encoder binary (default:{DEFAULT_ENCODER_DUT})") - parser.add_argument("--decdut", help=f"DUT decoder binary (default:{DEFAULT_DECODER_DUT})") - - args = parser.parse_args(argv[1:]) - - # check for DUT binaries - if args.encdut: - encdut_path = os.path.realpath(args.encdut) - if not os.path.exists(encdut_path): - sys.exit(f"DUT encoder binary {encdut_path} does not exist.") - else: - encdut_path = DEFAULT_ENCODER_DUT - if args.decdut: - decdut_path = os.path.realpath(args.decdut) - if not os.path.exists(decdut_path): - sys.exit(f"DUT encoder binary {decdut_path} does not exist.") - else: - decdut_path = DEFAULT_DECODER_DUT - if not os.path.exists(encdut_path) or not os.path.exists(decdut_path): - build_dut_binaries() - - if not os.path.exists(REFERENCE_DIR): - # check for REF binaries - if args.encref: - encref_path = os.path.realpath(args.encref) - if not os.path.exists(encref_path): - sys.exit(f"REF encoder binary {encref_path} does not exist.") - else: - encref_path = DEFAULT_ENCODER_REF - if args.decref: - decref_path = os.path.realpath(args.decref) - if not os.path.exists(decref_path): - sys.exit(f"REF encoder binary {decref_path} does not exist.") - else: - decref_path = DEFAULT_DECODER_REF - if not os.path.exists(encref_path) or not os.path.exists(decref_path): - sys.exit("Reference binaries do not exist.") - - # check for test vectors - if not os.path.exists(TEST_VECTOR_DIR): - sys.exit(f"Test vector directory {TEST_VECTOR_DIR} does not exist.") - - # check for references - if os.path.exists(REFERENCE_DIR): - print(f"Using existing references directory {REFERENCE_DIR}") - else: - # create references - print(f"Creating references within the references directory {REFERENCE_DIR}") - create_short_testvectors() - if platform.system() == "Windows": - base_cmd = ["pytest"] - else: - base_cmd = ["python3", "-m", "pytest"] - base_cmd += [ - "tests", - "-n", - args.numprocesses, - "--update_ref", - "1", - "-v", - "--test_vector_path", - TEST_VECTOR_DIR, - "--reference_path", - REFERENCE_DIR, - "--dut_base_path", - DUT_BASE_DIR, - "--ref_encoder_path", - encref_path, - "--ref_decoder_path", - decref_path, - "--dut_encoder_path", - encdut_path, - "--dut_decoder_path", - decdut_path, - ] - # work-around in unit tests via environment variable - # TESTVECTOR_PATH_REL_GROUPB: to specify the test vector directory relative to ivas_pytests folder - # TESTVECTOR_PATH_REL_TRUNK: to specify the test vector directory relative to trunk - my_env = os.environ.copy() - my_env["TESTVECTOR_PATH_REL_GROUPB"] = "testv/" - my_env["TESTVECTOR_PATH_REL_TRUNK"] = "/scripts/ivas_pytests/testv/" # leading "/" is important - my_env["CREND_UNIT_TEST_BIN"] = CREND_UNITTEST_REF - print("pytest command line to be executed from project root folder:") - print(" ".join(base_cmd + ["-m", "create_ref"])) - subprocess.run(base_cmd + ["-m", "create_ref"], check=False, env=my_env) - print("pytest command line to be executed from project root folder:") - print(" ".join(base_cmd + ["-m", "create_ref_part2"])) - subprocess.run(base_cmd + ["-m", "create_ref_part2"], check=False, env=my_env) - - if args.create_only: - return - - # run pytest - if platform.system() == "Windows": - cmd = ["pytest"] - else: - cmd = ["python3", "-m", "pytest"] - cmd += [ - "tests", - "-n", - args.numprocesses, - "-v", - "--test_vector_path", - TEST_VECTOR_DIR, - "--reference_path", - REFERENCE_DIR, - "--dut_base_path", - DUT_BASE_DIR, - "--dut_encoder_path", - encdut_path, - "--dut_decoder_path", - decdut_path, - "--junit-xml=report-junit.xml", - ] - # print pytest commandline - print("pytest command line to be executed from project root folder:") - print(" ".join(cmd)) - result = subprocess.run(cmd, check=False) - return result.returncode - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) -- GitLab From 6fa1b9f7105f13684aa7060c06c1f3dc10c24664 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Mon, 19 Sep 2022 18:34:52 +0200 Subject: [PATCH 14/25] pytest: removed pinning of python modules to specific versions in requirements.txt (mentioned versions have small bugs and latest versions seem to work fine) --- tests/requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 764694dfc0..2eb090f4fb 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,4 +1,4 @@ -pytest==5.3.5 -pytest-xdist==1.31.0 -scipy==1.5.2 -numpy==1.19.2 +pytest>=5.3.5 +pytest-xdist>=1.31.0 +scipy>=1.5.2 +numpy>=1.19.2 -- GitLab From 9ef8ba2743eed4a036162e70317e5c4b87b9dfa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Mon, 19 Sep 2022 18:39:10 +0200 Subject: [PATCH 15/25] pytest: update of README.md (addressed feedback, fixed typos and added custom options) --- tests/README.md | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/tests/README.md b/tests/README.md index dc22492fae..78ef9b2212 100644 --- a/tests/README.md +++ b/tests/README.md @@ -47,18 +47,20 @@ python3 tests/create_short_testvectors.py ``` The tests rely on references which need to be generated upfront using reference binaries. -When the reference binaries are named `IVAS_cod_ref(.exe)` and `IVAS_dec_ref.(exe)`, pytest will find and use them. +When the reference binaries are named `IVAS_cod_ref(.exe)` and `IVAS_dec_ref(.exe)`, pytest will find and use them. When the reference binaries are named differently, you need to specify them via the `--ref_encoder_path` and `--ref_decoder_path` options. -The tests will used the binaries `IVAS_cod(.exe)` and `IVAS_dec.(exe)` for testing. Please make sure that the binaries have been built before running the tests. +The tests will used the binaries `IVAS_cod(.exe)` and `IVAS_dec(.exe)` for testing. Please make sure that the binaries have been built before running the tests. +When different test binaries are to be used, they can be specified via the `--dut_encoder_path` and `--dut_decoder_path` options. +(DUT: Device Under Test) ```bash # create references -# the following binaies need to be present: +# the following binaries need to be present: # - IVAS_cod(.exe) -# - IVAS_dec.(exe) +# - IVAS_dec(.exe) # - IVAS_cod_ref(.exe) -# - IVAS_dec_ref.(exe) +# - IVAS_dec_ref(.exe) # pytest command lines to be executed from project root folder: pytest tests -n auto --update_ref 1 -m create_ref pytest tests -n auto --update_ref 1 -m create_ref_part2 @@ -105,3 +107,23 @@ When there a many test failures, you can use the `-x` (or `--exitfirst`) option Commonly used options like `-n auto` can be added to addopts within the [pytest] section in `pytest.ini`. This saves some typing when calling `pytest`. The `-v` (or `--verbose`) option is usually helpful to see what is going on. Therefore, `-v` is currently part of addopts in `pytest.ini`. If you don't like this verbosity, you can specify the `-q` (`--quiet`) option when running `pytest`. + +## Custom options + +`Note:` +The custom options are listed as part of the pytest help `pytest -h`. + +```text +--update_ref=UPDATE_REF Indicate whether references shall be updated. + 0: Only DUT processing, no reference generation, references need to be present. + 1: Only reference generation (unconditionally), no DUT processing. + 2: DUT processing, references are generated when not present (not supported by all tests). +--dut_encoder_path=DUT_ENCODER_PATH If specified, use given binary as DUT encoder. +--dut_decoder_path=DUT_DECODER_PATH If specified, use given binary as DUT decoder. +--ref_encoder_path=REF_ENCODER_PATH If specified, use given binary as REF encoder. +--ref_decoder_path=REF_DECODER_PATH If specified, use given binary as REF decoder. +--test_vector_path=TEST_VECTOR_PATH If specified, use given directory as base directory for test vector files. +--reference_path=REFERENCE_PATH If specified, use given directory as base directory for reference files. +--dut_base_path=DUT_BASE_PATH If specified, use given directory as base data directory for dut files. +--param_file=PARAM_FILE If specified, use given param file in test_param_file. +``` -- GitLab From bb2e79c3d97f93a3ae4e650e708a78c9fe5627dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Mon, 19 Sep 2022 18:42:06 +0200 Subject: [PATCH 16/25] pytest: update of .gitignore and pytest.ini (adjusted JUnit reporting) --- .gitignore | 2 ++ pytest.ini | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 9000be58a1..aecd59fd31 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,8 @@ scripts/ref/ scripts/test/ scripts/out/ scripts/self_test_summary.txt +tests/dut +tests/ref # Python files that pop up when running scripts __pycache__/ diff --git a/pytest.ini b/pytest.ini index 9d9a3bbf8e..9ffbd0f806 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,7 +2,12 @@ # note: per convention, this file is placed in the root directory of the repository [pytest] addopts = -ra --tb=short --basetemp=./tmp -v -junit_family=xunit1 +# Write captured system-out log messages to JUnit report. +junit_logging = system-out +# Do not capture log information for passing tests to JUnit report. +junit_log_passing_tests = False +junit_duration_report = call +junit_family = xunit1 log_file_level = DEBUG log_format = %(asctime)s %(levelname)s %(message)s log_date_format = %Y-%m-%d %H:%M:%S -- GitLab From 4d7905745114f2fc9fc94d5d5888788a4ddb3350 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Wed, 21 Sep 2022 17:18:26 +0200 Subject: [PATCH 17/25] fix junit entry to get both reports --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a6c4600fce..aa0d518a93 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -355,8 +355,9 @@ pytest-on-merge-request: - report-junit-evs.xml expose_as: 'pytest results' reports: - junit: report-junit.xml - junit: report-junit-evs.xml + junit: + - report-junit.xml + - report-junit-evs.xml # --------------------------------------------------------------- -- GitLab From 017cb80fb3903a4301b37c0c68ecd593900378a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Mon, 26 Sep 2022 17:06:42 +0200 Subject: [PATCH 18/25] pytest reporting improvements - cmp_custom() returns reason (str) in addition to retval (int) - reason is part of the assert and will show up in the short test summary - SBA tests now do a single comparison on the interleaved PCM; il2mm.py is removed --- tests/cmp_custom.py | 36 ++++++++++----------- tests/il2mm.py | 61 ------------------------------------ tests/test_param_file.py | 4 +-- tests/test_sba_bs_dec_plc.py | 45 ++++++++++---------------- tests/test_sba_bs_enc.py | 52 ++++++++++-------------------- 5 files changed, 53 insertions(+), 145 deletions(-) delete mode 100644 tests/il2mm.py diff --git a/tests/cmp_custom.py b/tests/cmp_custom.py index 1999d7ccc5..408d6408cd 100755 --- a/tests/cmp_custom.py +++ b/tests/cmp_custom.py @@ -89,7 +89,7 @@ class CompareSamples: self.file_1.seek(0) self.file_2.seek(0) - def print_summary(self): + def print_summary(self) -> (int, str): """ Print the summary of the comparison. """ @@ -105,19 +105,18 @@ class CompareSamples: if not self.diff_present: print("Comparison success") print("") - return 0 - else: - print( - f"First unmatched diff ==> {self.first_diff}", - f"at sample num {self.first_diff_sample_num}", - ) - print( - f"MAXIMUM ABS DIFF ==> {self.max_diff} at sample num {self.max_diff_sample_num}" - ) - print("Comparison failed") - print("") - return 1 - return 1 + return 0, "Comparison success" + + # comparison failed + print( + f"First unmatched diff ==> {self.first_diff}", + f"at sample num {self.first_diff_sample_num}", + ) + diff_msg = f"MAXIMUM ABS DIFF ==> {self.max_diff} at sample num {self.max_diff_sample_num}" + print(diff_msg) + print("Comparison failed") + print("") + return 1, f"Comparison failed, {diff_msg}" def compare_next_sample(self): """ @@ -148,7 +147,7 @@ class CompareSamples: def usage(): print(__doc__) - return 1 + return 1, "" def cmp_custom( @@ -157,7 +156,7 @@ def cmp_custom( sample_size_in_bytes_str, tolerance_str, end_samples_to_skip_str="0", -): +) -> (int, str): """ Function to compare the samples in 2 PCM files. """ @@ -191,10 +190,11 @@ def cmp_custom( return cmp_samples.print_summary() -def main(argv): +def main(argv) -> int: if len(argv) < 5: return usage() - return cmp_custom(*argv[1:]) + retval, reason = cmp_custom(*argv[1:]) + return retval if __name__ == "__main__": diff --git a/tests/il2mm.py b/tests/il2mm.py deleted file mode 100644 index eb09593bdb..0000000000 --- a/tests/il2mm.py +++ /dev/null @@ -1,61 +0,0 @@ -""" - (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. -""" - -import os - - -def il2mm(file_in, num_ch, b_delete=True): - """ - Convert interleaved input file to multiple mono output files. - """ - num_bytes_per_sample = 2 - num_bytes_per_frame = num_bytes_per_sample * num_ch - num_bytes_per_channel = os.path.getsize(file_in) / num_ch - - with open(file_in, "rb") as fid_in: - out_path = os.path.splitext(file_in)[0] - for chan in range(num_ch): - file_out = out_path + str(chan + 1) + "ch.raw" - with open(file_out, "wb") as fid_out: - bytes_written = 0 - offset = chan * num_bytes_per_sample - fid_in.seek(offset, 0) - while bytes_written < num_bytes_per_channel: - data = fid_in.read(num_bytes_per_sample) - fid_in.seek(num_bytes_per_frame - num_bytes_per_sample, 1) - written = fid_out.write(bytes(data)) - assert ( - written == num_bytes_per_sample - ), f"Error writing data: {written} != {num_bytes_per_sample}" - bytes_written += num_bytes_per_sample - - # delete interleaved input file - if b_delete: - os.remove(file_in) diff --git a/tests/test_param_file.py b/tests/test_param_file.py index be4ffe62ec..d376f93bb8 100644 --- a/tests/test_param_file.py +++ b/tests/test_param_file.py @@ -317,5 +317,5 @@ def compare( """ sample_size = "2" # 16-bit samples tolerance = "0" # zero tolerance for BE testing - cmp_result = cmp_custom(pcm_file_1, pcm_file_2, sample_size, tolerance) - assert cmp_result == 0 + cmp_result, reason = cmp_custom(pcm_file_1, pcm_file_2, sample_size, tolerance) + assert cmp_result == 0, reason diff --git a/tests/test_sba_bs_dec_plc.py b/tests/test_sba_bs_dec_plc.py index b76975c28e..aa317fa07f 100644 --- a/tests/test_sba_bs_dec_plc.py +++ b/tests/test_sba_bs_dec_plc.py @@ -30,10 +30,8 @@ import os import pytest -import shutil import errno -from il2mm import il2mm from cmp_custom import cmp_custom from conftest import DecoderFrontend @@ -45,7 +43,6 @@ ivas_br_list = ['32000', '64000', '96000', '256000'] sampling_rate_list = ['48', '32', '16'] agc_list = [0, 1] -ch_count_foa = 4 AbsTol = '3' @@ -88,7 +85,7 @@ def test_sba_plc_system( tag = tag + fs + 'c' #dec - sba_dec_plc(dut_decoder_frontend, test_vector_path, reference_path, dut_base_path, ref_decoder_path, tag, ch_count_foa, fs, ivas_br, dtx, plc_pattern, update_ref, agc) + sba_dec_plc(dut_decoder_frontend, test_vector_path, reference_path, dut_base_path, ref_decoder_path, tag, fs, ivas_br, dtx, plc_pattern, update_ref, agc) ######################################################### @@ -100,7 +97,6 @@ def sba_dec_plc( dut_base_path, ref_decoder_path, tag, - ch_count, sampling_rate, ivas_br, dtx, @@ -116,8 +112,8 @@ def sba_dec_plc( tag_out += '_AGC1' plc_tag_out = f"{tag_out}_{plc_pattern}" - dut_out_dir = f"{dut_base_path}/sba_bs/raw/{plc_tag_out}" - ref_out_dir = f"{reference_path}/sba_bs/raw/{plc_tag_out}" + dut_out_dir = f"{dut_base_path}/sba_bs/raw" + ref_out_dir = f"{reference_path}/sba_bs/raw" check_and_makedir(dut_out_dir) check_and_makedir(ref_out_dir) @@ -126,6 +122,9 @@ def sba_dec_plc( ref_in_pkt = f"{reference_path}/sba_bs/pkt/{tag_out}.pkt" ref_in_pkt_dutenc = f"{reference_path}/sba_bs/pkt/{tag_out}_dutenc.pkt" + dut_out_raw = f"{dut_out_dir}/{plc_tag_out}.raw" + ref_out_raw = f"{ref_out_dir}/{plc_tag_out}.raw" + if ref_decoder_path: ref_decoder = DecoderFrontend(ref_decoder_path, "REF") @@ -134,44 +133,34 @@ def sba_dec_plc( "FOA", sampling_rate, ref_in_pkt, - f"{ref_out_dir}/out.raw", + ref_out_raw, plc_file=plc_file, ) - # convert REF interleaved to multi-mono - il2mm(f"{ref_out_dir}/out.raw", ch_count) - if update_ref == 0: # call DUT decoder decoder_frontend.run( "FOA", sampling_rate, ref_in_pkt_dutenc, - f"{dut_out_dir}/out.raw", + dut_out_raw, plc_file=plc_file, ) - il2mm(f"{dut_out_dir}/out.raw", ch_count) - ######### compare cmd ##################################### end_skip_samples = '0' - test_fail = False - for count in range(ch_count): - ch_id = str(count + 1) - - if cmp_custom( - f"{dut_out_dir}/out{ch_id}ch.raw", - f"{ref_out_dir}/out{ch_id}ch.raw", - "2", - AbsTol, - end_skip_samples - ) != 0: - test_fail = True + cmp_result, reason = cmp_custom( + dut_out_raw, + ref_out_raw, + "2", + AbsTol, + end_skip_samples + ) ##File removal## - shutil.rmtree(dut_out_dir, ignore_errors=True) + os.remove(dut_out_raw) ##report failure - assert not test_fail + assert cmp_result == 0, reason diff --git a/tests/test_sba_bs_enc.py b/tests/test_sba_bs_enc.py index 3a37b5db42..15973b5332 100644 --- a/tests/test_sba_bs_enc.py +++ b/tests/test_sba_bs_enc.py @@ -36,9 +36,7 @@ The outputs are compared with C generated references. import os import pytest import errno -import shutil -from il2mm import il2mm from cmp_custom import cmp_custom from cut_pcm import cut_samples from conftest import EncoderFrontend, DecoderFrontend @@ -64,7 +62,6 @@ agc_list = [0, 1] sample_rate_bw_idx_list = [('48', 'SWB'), ('48', 'WB'), ('32', 'WB')] AbsTol = '4' -ch_count_foa = 4 def check_and_makedir(dir_path): @@ -129,7 +126,6 @@ def test_bypass_enc( reference_path, dut_base_path, tag, - ch_count_foa, fs, ivas_br, dtx, @@ -199,7 +195,6 @@ def test_sba_enc_system( reference_path, dut_base_path, tag, - ch_count_foa, fs, ivas_br, dtx, @@ -261,7 +256,6 @@ def test_spar_hoa2_enc_system( reference_path, dut_base_path, tag, - ch_count_foa, fs, ivas_br, dtx, @@ -323,7 +317,6 @@ def test_spar_hoa3_enc_system( reference_path, dut_base_path, tag, - ch_count_foa, fs, ivas_br, dtx, @@ -386,7 +379,6 @@ def test_sba_enc_BWforce_system( reference_path, dut_base_path, tag, - ch_count_foa, fs, ivas_br, dtx, @@ -515,7 +507,6 @@ def sba_dec( reference_path, dut_base_path, tag, - ch_count, sampling_rate, ivas_br, dtx, @@ -546,12 +537,15 @@ def sba_dec( # to avoid conflicting names in case of parallel test execution, differentiate all cases long_tag_ext = f"_AGC{agc}_pca{bypass}" - dut_out_dir = f"{dut_base_path}/sba_bs/raw/{tag_out}{long_tag_ext}" - ref_out_dir = f"{reference_path}/sba_bs/raw/{tag_out}{short_tag_ext}" + dut_out_dir = f"{dut_base_path}/sba_bs/raw" + ref_out_dir = f"{reference_path}/sba_bs/raw" dut_in_pkt = f"{dut_base_path}/sba_bs/pkt/{tag_out}{long_tag_ext}.pkt" ref_in_pkt = f"{reference_path}/sba_bs/pkt/{tag_out}{short_tag_ext}.pkt" + dut_out_raw = f"{dut_out_dir}/{tag_out}{long_tag_ext}.raw" + ref_out_raw = f"{ref_out_dir}/{tag_out}{short_tag_ext}.raw" + check_and_makedir(dut_out_dir) check_and_makedir(ref_out_dir) @@ -563,48 +557,34 @@ def sba_dec( output_config, sampling_rate, ref_in_pkt, - f"{ref_out_dir}/out.raw", + ref_out_raw, ) - # convert REF interleaved to multi-mono - il2mm(f"{ref_out_dir}/out.raw", ch_count, b_delete=not keep_files) - if update_ref == 0: # call DUT decoder decoder_frontend.run( output_config, sampling_rate, dut_in_pkt, - f"{dut_out_dir}/out.raw", + dut_out_raw, ) - il2mm(f"{dut_out_dir}/out.raw", ch_count, b_delete=not keep_files) - ######### compare cmd ##################################### end_skip_samples = '0' - test_fail = False - for count in range(ch_count): - ch_id = str(count + 1) - - # TEST - fsize1 = os.path.getsize(f"{dut_out_dir}/out{ch_id}ch.raw") - fsize2 = os.path.getsize(f"{ref_out_dir}/out{ch_id}ch.raw") - print(f"Want to compare {dut_out_dir}/out{ch_id}ch.raw ({fsize1} bytes) with {ref_out_dir}/out{ch_id}ch.raw ({fsize2} bytes)") - if cmp_custom( - f"{dut_out_dir}/out{ch_id}ch.raw", - f"{ref_out_dir}/out{ch_id}ch.raw", - "2", - AbsTol, - end_skip_samples - ) != 0: - test_fail = True + cmp_result, reason = cmp_custom( + dut_out_raw, + ref_out_raw, + "2", + AbsTol, + end_skip_samples + ) ##File removal## if not keep_files: os.remove(dut_in_pkt) - shutil.rmtree(dut_out_dir, ignore_errors=True) + os.remove(dut_out_raw) ##report failure - assert not test_fail + assert cmp_result == 0, reason -- GitLab From 7638cd85538a441e7da913d70399467a308db9b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Thu, 29 Sep 2022 10:43:51 +0200 Subject: [PATCH 19/25] pytest: add network simulator call to fix new JBM tests --- tests/test_param_file.py | 142 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 134 insertions(+), 8 deletions(-) diff --git a/tests/test_param_file.py b/tests/test_param_file.py index d376f93bb8..e2b1ba7aa1 100644 --- a/tests/test_param_file.py +++ b/tests/test_param_file.py @@ -31,11 +31,29 @@ the United Nations Convention on Contracts on the International Sales of Goods. import os import errno import pytest +import platform +from subprocess import run from cmp_custom import cmp_custom from conftest import EncoderFrontend, DecoderFrontend from testconfig import PARAM_FILE +VALID_DEC_OUTPUT_CONF = [ + "MONO", + "STEREO", + "5_1", + "7_1", + "5_1_2", + "5_1_4", + "7_1_4", + "FOA", + "HOA2", + "HOA3", + "BINAURAL", + "BINAURAL_ROOM", + "EXT", +] + param_file_test_dict = {} with open(PARAM_FILE, "r", encoding="UTF-8") as fp: data = fp.read() @@ -44,6 +62,7 @@ with open(PARAM_FILE, "r", encoding="UTF-8") as fp: tag = "" enc_opts = "" dec_opts = "" + sim_opts = "" for line in block.split("\n"): if line.startswith("// "): tag = line[3:] @@ -51,13 +70,15 @@ with open(PARAM_FILE, "r", encoding="UTF-8") as fp: enc_opts = line[12:] if line.startswith("../IVAS_dec "): dec_opts = line[12:] + if line.startswith("networkSimulator_g192 "): + sim_opts = line[22:] if tag == "" or enc_opts == "" or dec_opts == "": # no complete parameter set continue if tag in param_file_test_dict: print("non-unique tag found - ignoring new entry") continue - param_file_test_dict[tag] = (enc_opts, dec_opts) + param_file_test_dict[tag] = (enc_opts, dec_opts, sim_opts) def check_and_makedir(dir_path): @@ -101,9 +122,10 @@ def test_param_file_tests( dut_base_path, test_vector_path, update_ref, + rootdir, test_tag, ): - enc_opts, dec_opts = param_file_test_dict[test_tag] + enc_opts, dec_opts, sim_opts = param_file_test_dict[test_tag] tag_str = convert_test_string_to_tag(test_tag) @@ -148,6 +170,36 @@ def test_param_file_tests( update_ref, ) + # check for networkSimulator_g192 command line + if sim_opts != "": + sim_split = sim_opts.split() + assert len(sim_split) == 6, "networkSimulator_g192 binary expects 6 parameters" + # [sim_profile, sim_input, sim_output, sim_trace, sim_nFPP, sim_offset] = sim_split + if sim_split[0].startswith(("../")): + # remove leading "../" + sim_split[0] = sim_split[0][3:] + assert sim_split[1] == "bit" + # in the parameter file, only "bit" is used as bitstream file name + # -> re-use bitstream filename from encoder call + sim_split[1] = bitstream_file + assert sim_split[2] == "netsimoutput" + # in the parameter file, only "netsimoutput" is used as netsim output file name + # -> construct netsim output file name + netsim_outfile = f"{testv_base}_{tag_str}.netsimout" + sim_split[2] = netsim_outfile + assert sim_split[3] == "tracefile_sim" + # in the parameter file, only "tracefile_sim" is used as trace output file name + # -> construct trace output file name + netsim_trace_outfile = f"{testv_base}_{tag_str}.netsimtrace" + sim_split[3] = netsim_trace_outfile + simulate( + reference_path, + dut_base_path, + sim_split, + update_ref, + rootdir, + ) + # evaluate decoder options dec_split = dec_opts.split() assert len(dec_split) >= 3 @@ -165,6 +217,11 @@ def test_param_file_tests( sampling_rate = int(dec_split.pop()) if len(dec_split) > 0: output_config = dec_split.pop() + if output_config.upper() not in VALID_DEC_OUTPUT_CONF: + if not output_config.endswith(".txt"): + # must be EVS tests with additional parameters - put param back + dec_split.append(output_config) + output_config = "" else: output_config = "" @@ -173,9 +230,17 @@ def test_param_file_tests( # the output config is a file output_config_name = os.path.splitext(os.path.basename(output_config))[0] - assert bitstream_file_dec == "bit" - # in the parameter file, only "bit" is used as bitstream file name - # -> re-use bitstream filename from encoder call + tracefile_dec = "" + if sim_opts != "": + assert bitstream_file_dec == "netsimoutput" + # in the parameter file, only "netsimoutput" is used as bitstream file name + # -> re-use netsim_outfile + bitstream_file = netsim_outfile + tracefile_dec = f"{testv_base}_{tag_str}.dectrace" + else: + assert bitstream_file_dec == "bit" + # in the parameter file, only "bit" is used as bitstream file name + # -> re-use bitstream filename from encoder call # the output file is not the real output filename # -> construct output filename @@ -196,6 +261,7 @@ def test_param_file_tests( output_file, dec_split, update_ref, + tracefile_dec, ) # compare @@ -222,7 +288,7 @@ def encode( update_ref, ): """ - Call REF and/or DUT decoder. + Call REF and/or DUT encoder. """ # directories dut_out_dir = f"{dut_base_path}/param_file/enc" @@ -256,6 +322,53 @@ def encode( ) +def simulate( + reference_path, + dut_base_path, + sim_opts_list, + update_ref, + rootdir, +): + """ + Call network simulator on REF and/or DUT encoder output. + """ + # directories + dut_out_dir = f"{dut_base_path}/param_file/enc" + ref_out_dir = f"{reference_path}/param_file/enc" + + netsim_infile = sim_opts_list[1] + netsim_outfile = sim_opts_list[2] + netsim_tracefile = sim_opts_list[3] + ref_out_file = f"{ref_out_dir}/{netsim_outfile}" + + if platform.system() == "Windows": + netsim = [os.path.join(rootdir, "scripts", "tools", "Win32", "networkSimulator_g192.exe")] + elif platform.system() == "Linux": + # there is no Linux binary available -> use the Win32 binary via wine + netsim = [ + "wine", + os.path.join(rootdir, "scripts", "tools", "Win32", "networkSimulator_g192.exe"), + ] + elif platform.system() == "Darwin": + netsim = [os.path.join(rootdir, "scripts", "tools", "Darwin", "networkSimulator_g192")] + + if update_ref == 1 or update_ref == 2 and not os.path.exists(ref_out_file): + # call network simulator on REF encoder output + cmd_opts = sim_opts_list + cmd_opts[1] = f"{ref_out_dir}/{netsim_infile}" + cmd_opts[2] = f"{ref_out_dir}/{netsim_outfile}" # ref_out_file + cmd_opts[3] = f"{ref_out_dir}/{netsim_tracefile}" + run(netsim + cmd_opts, check=False) + + if update_ref in [0, 2]: + # call network simulator on DUT encoder output + cmd_opts = sim_opts_list + cmd_opts[1] = f"{dut_out_dir}/{netsim_infile}" + cmd_opts[2] = f"{dut_out_dir}/{netsim_outfile}" # dut_out_file + cmd_opts[3] = f"{dut_out_dir}/{netsim_tracefile}" + run(netsim + cmd_opts, check=False) + + def decode( decoder_frontend, ref_decoder_path, @@ -267,6 +380,7 @@ def decode( output_file, dec_opts_list, update_ref, + tracefile_dec, ): """ Call REF and/or DUT decoder. @@ -282,6 +396,12 @@ def decode( if update_ref == 1 or update_ref == 2 and not os.path.exists(ref_out_file): check_and_makedir(ref_out_dir) + add_option_list = dec_opts_list + if tracefile_dec != "": + add_option_list = [ + x if x != "tracefile_dec" else f"{ref_out_dir}/{tracefile_dec}" + for x in dec_opts_list + ] # call REF decoder assert ref_decoder_path ref_decoder = DecoderFrontend(ref_decoder_path, "REF") @@ -290,18 +410,24 @@ def decode( sampling_rate, ref_in_file, ref_out_file, - add_option_list=dec_opts_list, + add_option_list=add_option_list, ) if update_ref in [0, 2]: check_and_makedir(dut_out_dir) + add_option_list = dec_opts_list + if tracefile_dec != "": + add_option_list = [ + x if x != "tracefile_dec" else f"{dut_out_dir}/{tracefile_dec}" + for x in dec_opts_list + ] # call DUT decoder decoder_frontend.run( output_config, sampling_rate, dut_in_file, dut_out_file, - add_option_list=dec_opts_list, + add_option_list=add_option_list, ) -- GitLab From f55e8847ced033209274be7dc4a4cdd7812777ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Thu, 29 Sep 2022 12:04:53 +0200 Subject: [PATCH 20/25] pytest: change default options (add '-n auto', remove '-v') --- .gitlab-ci.yml | 16 ++++++++-------- pytest.ini | 2 +- tests/README.md | 12 ++++++------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8bea76dedd..0f62a3ab76 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -315,13 +315,13 @@ pytest-on-merge-request: - mv IVAS_cod_test IVAS_cod - mv IVAS_dec_test IVAS_dec # create references - - python3 -m pytest tests -n auto --update_ref 1 -m create_ref - - python3 -m pytest tests -n auto --update_ref 1 -m create_ref_part2 - - python3 -m pytest tests/test_param_file.py -n auto --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm + - python3 -m pytest tests -v --update_ref 1 -m create_ref + - python3 -m pytest tests -v --update_ref 1 -m create_ref_part2 + - python3 -m pytest tests/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm ### run pytest - exit_code=0 - - python3 -m pytest tests -n auto --junit-xml=report-junit.xml || exit_code=$? + - python3 -m pytest tests -v --junit-xml=report-junit.xml || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 1 ]; then echo "Run errors in pytest"; fail_1=1; fi @@ -331,7 +331,7 @@ pytest-on-merge-request: ### run pytest for EVS cases - exit_code=0 - - python3 -m pytest tests/test_param_file.py -n auto --param_file scripts/config/self_test_evs.prm --junit-xml=report-junit-evs.xml || exit_code=$? + - python3 -m pytest tests/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --junit-xml=report-junit-evs.xml || exit_code=$? - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 1 ]; then echo "Run errors in pytest for EVS"; fail_2=1; fi @@ -444,12 +444,12 @@ codec-comparison-on-main-push: - mv IVAS_cod_test IVAS_cod - mv IVAS_dec_test IVAS_dec # create references - - python3 -m pytest tests -n auto --update_ref 1 -m create_ref - - python3 -m pytest tests -n auto --update_ref 1 -m create_ref_part2 + - python3 -m pytest tests -v --update_ref 1 -m create_ref + - python3 -m pytest tests -v --update_ref 1 -m create_ref_part2 ### run pytest - exit_code=0 - - python3 -m pytest tests -n auto --junit-xml=report-junit.xml || exit_code=$? + - python3 -m pytest tests -v --junit-xml=report-junit.xml || exit_code=$? - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi diff --git a/pytest.ini b/pytest.ini index 9ffbd0f806..c4ed77a9f6 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,7 +1,7 @@ # pytest.ini # note: per convention, this file is placed in the root directory of the repository [pytest] -addopts = -ra --tb=short --basetemp=./tmp -v +addopts = -ra --tb=short --basetemp=./tmp -n auto # Write captured system-out log messages to JUnit report. junit_logging = system-out # Do not capture log information for passing tests to JUnit report. diff --git a/tests/README.md b/tests/README.md index 78ef9b2212..12f0fe73f5 100644 --- a/tests/README.md +++ b/tests/README.md @@ -62,8 +62,8 @@ When different test binaries are to be used, they can be specified via the `--du # - IVAS_cod_ref(.exe) # - IVAS_dec_ref(.exe) # pytest command lines to be executed from project root folder: -pytest tests -n auto --update_ref 1 -m create_ref -pytest tests -n auto --update_ref 1 -m create_ref_part2 +pytest tests --update_ref 1 -m create_ref +pytest tests --update_ref 1 -m create_ref_part2 ``` ## Running the tests @@ -72,7 +72,7 @@ To run all tests from the tests folder: ```bash # pytest command line to be executed from project root folder: -pytest tests -n auto +pytest tests ``` ## Re-running some tests @@ -81,7 +81,7 @@ When there are test failures, you may want to run, after having fixed the code, ```bash # rerun only the tests that failed at the last run -pytest tests -n auto --last-failed +pytest tests --last-failed ``` To run a specific test case, you can e.g. pick a test case from the `short test summary info` and use that test case as an argument to `pytest`. E.g. @@ -104,9 +104,9 @@ pytest tests/test_sba_bs_dec_plc.py::test_sba_plc_system When there a many test failures, you can use the `-x` (or `--exitfirst`) option to stop testing on the first failure. -Commonly used options like `-n auto` can be added to addopts within the [pytest] section in `pytest.ini`. This saves some typing when calling `pytest`. +Commonly used options like `-n auto` are added to addopts within the [pytest] section in `pytest.ini`. This saves some typing when calling `pytest`. -The `-v` (or `--verbose`) option is usually helpful to see what is going on. Therefore, `-v` is currently part of addopts in `pytest.ini`. If you don't like this verbosity, you can specify the `-q` (`--quiet`) option when running `pytest`. +The `-v` (or `--verbose`) option is helpful to see what is going on. ## Custom options -- GitLab From 629a214804ed5a45a79243e47c5380b55b18b1b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Thu, 29 Sep 2022 16:18:54 +0200 Subject: [PATCH 21/25] pytest: adjust coverage-test-on-main-scheduled to use pytest --- .gitlab-ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0f62a3ab76..a16cdc3666 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -629,9 +629,10 @@ coverage-test-on-main-scheduled: script: - *print-common-info - make GCOV=1 -j - - ./scripts/self_test.py --create -t 1 - - ./scripts/self_test.py --create -t 1 scripts/config/self_test_evs.prm - - ./scripts/ivas_pytests/self_test_b.py --create_only --numprocesses 1 --encref IVAS_cod --decref IVAS_dec --encdut IVAS_cod --decdut IVAS_dec + - python3 tests/create_short_testvectors.py + - python3 -m pytest tests -v -n 0 --update_ref 1 -m create_ref --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec + - python3 -m pytest tests -v -n 0 --update_ref 1 -m create_ref_part2 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec + - python3 -m pytest tests/test_param_file.py -v -n 0 --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec - lcov -c -d obj -o coverage.info - genhtml coverage.info -o coverage artifacts: -- GitLab From 8c63cd97dbb4fa0ec03a60aa522b4724c9d8b131 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Thu, 29 Sep 2022 19:50:42 +0200 Subject: [PATCH 22/25] pytest: add --keep_files option --- tests/README.md | 2 ++ tests/conftest.py | 20 +++++++++++++++++ tests/test_param_file.py | 22 ++++++++++++------ tests/test_sba_bs_dec_plc.py | 38 ++++++++++++++++++++++++------- tests/test_sba_bs_enc.py | 43 ++++++++++++++++++++++-------------- 5 files changed, 94 insertions(+), 31 deletions(-) diff --git a/tests/README.md b/tests/README.md index 12f0fe73f5..470542760d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -126,4 +126,6 @@ The custom options are listed as part of the pytest help `pytest -h`. --reference_path=REFERENCE_PATH If specified, use given directory as base directory for reference files. --dut_base_path=DUT_BASE_PATH If specified, use given directory as base data directory for dut files. --param_file=PARAM_FILE If specified, use given param file in test_param_file. +--keep_files By default, the DUT output files of successful tests are deleted. + Use --keep_files to prevent these deletions. ``` diff --git a/tests/conftest.py b/tests/conftest.py index c49b6d3d83..1decc4c7f7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,6 +28,11 @@ the United Nations Convention on Contracts on the International Sales of Goods. """ +__doc__ = \ +""" +Pytest customization (configuration and fixtures) for the IVAS codec test suite. +""" + import logging from pathlib import Path import platform @@ -120,6 +125,13 @@ def pytest_addoption(parser): help="If specified, use given param file in test_param_file.", ) + parser.addoption( + "--keep_files", + action="store_true", + help="By default, the DUT output files of successful tests are deleted." + " Use --keep_files to prevent these deletions.", + ) + @pytest.fixture(scope="session", autouse=True) def update_ref(request): @@ -132,6 +144,14 @@ def update_ref(request): return int(request.config.getoption("--update_ref")) +@pytest.fixture(scope="session") +def keep_files(request) -> bool: + """ + Return indication to not delete DUT output files. + """ + return request.config.option.keep_files + + @pytest.fixture(scope="session") def dut_encoder_path(request) -> str: """ diff --git a/tests/test_param_file.py b/tests/test_param_file.py index e2b1ba7aa1..f690c67fec 100644 --- a/tests/test_param_file.py +++ b/tests/test_param_file.py @@ -1,7 +1,3 @@ -""" -Execute tests specified via a parameter file. -""" - __copyright__ = """ (C) 2022 Baseline Development Group with portions copyright Dolby International AB, Ericsson AB, Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., @@ -28,11 +24,16 @@ accordance with the laws of the Federal Republic of Germany excluding its confli the United Nations Convention on Contracts on the International Sales of Goods. """ +__doc__ = \ +""" +Execute tests specified via a parameter file. +""" + import os import errno -import pytest import platform from subprocess import run +import pytest from cmp_custom import cmp_custom from conftest import EncoderFrontend, DecoderFrontend from testconfig import PARAM_FILE @@ -123,6 +124,7 @@ def test_param_file_tests( test_vector_path, update_ref, rootdir, + keep_files, test_tag, ): enc_opts, dec_opts, sim_opts = param_file_test_dict[test_tag] @@ -271,8 +273,14 @@ def test_param_file_tests( f"{reference_path}/param_file/dec/{output_file}", ) - # clean-up - # TODO: consider removing DUT output files when test result is OK (to save disk space) + # remove DUT output files when test result is OK (to save disk space) + if not keep_files: + os.remove(f"{dut_base_path}/param_file/enc/{bitstream_file}") + os.remove(f"{dut_base_path}/param_file/dec/{output_file}") + if sim_opts != "": + os.remove(f"{dut_base_path}/param_file/enc/{testv_base}_{tag_str}.192") + os.remove(f"{dut_base_path}/param_file/enc/{netsim_trace_outfile}") + os.remove(f"{dut_base_path}/param_file/dec/{tracefile_dec}") def encode( diff --git a/tests/test_sba_bs_dec_plc.py b/tests/test_sba_bs_dec_plc.py index aa317fa07f..3922eed840 100644 --- a/tests/test_sba_bs_dec_plc.py +++ b/tests/test_sba_bs_dec_plc.py @@ -28,11 +28,16 @@ the United Nations Convention on Contracts on the International Sales of Goods. """ +__doc__ = \ +""" +Execute SBA decoder tests using different PLC patterns. +""" + import os -import pytest import errno +import pytest -from cmp_custom import cmp_custom +from cmp_custom import cmp_custom from conftest import DecoderFrontend #params @@ -75,6 +80,7 @@ def test_sba_plc_system( dut_base_path, ref_decoder_path, update_ref, + keep_files, ivas_br, dtx, tag, @@ -85,7 +91,21 @@ def test_sba_plc_system( tag = tag + fs + 'c' #dec - sba_dec_plc(dut_decoder_frontend, test_vector_path, reference_path, dut_base_path, ref_decoder_path, tag, fs, ivas_br, dtx, plc_pattern, update_ref, agc) + sba_dec_plc( + dut_decoder_frontend, + test_vector_path, + reference_path, + dut_base_path, + ref_decoder_path, + tag, + fs, + ivas_br, + dtx, + plc_pattern, + update_ref, + agc, + keep_files, + ) ######################################################### @@ -102,7 +122,8 @@ def sba_dec_plc( dtx, plc_pattern, update_ref, - agc + agc, + keep_files, ): ######### run cmd ##################################### @@ -159,8 +180,9 @@ def sba_dec_plc( end_skip_samples ) - ##File removal## - os.remove(dut_out_raw) - - ##report failure + # report compare result assert cmp_result == 0, reason + + # remove DUT output files when test result is OK (to save disk space) + if not keep_files: + os.remove(dut_out_raw) diff --git a/tests/test_sba_bs_enc.py b/tests/test_sba_bs_enc.py index 15973b5332..048a7e46ab 100644 --- a/tests/test_sba_bs_enc.py +++ b/tests/test_sba_bs_enc.py @@ -28,14 +28,15 @@ the United Nations Convention on Contracts on the International Sales of Goods. """ +__doc__ = \ """ Test file to run C encoder and decoder code. The outputs are compared with C generated references. """ import os -import pytest import errno +import pytest from cmp_custom import cmp_custom from cut_pcm import cut_samples @@ -86,9 +87,10 @@ def test_bypass_enc( ref_encoder_path, ref_decoder_path, update_ref, + keep_files, tag, fs, - bypass + bypass, ): if update_ref == 1 and bypass == 1: pytest.skip() @@ -133,7 +135,8 @@ def test_bypass_enc( bypass, agc, output_config, - update_ref + update_ref, + keep_files, ) @@ -152,11 +155,12 @@ def test_sba_enc_system( ref_encoder_path, ref_decoder_path, update_ref, + keep_files, ivas_br, dtx, tag, fs, - agc + agc, ): tag = tag + fs + 'c' max_bw = "FB" @@ -202,7 +206,8 @@ def test_sba_enc_system( bypass, agc, output_config, - update_ref + update_ref, + keep_files, ) @pytest.mark.create_ref @@ -217,8 +222,9 @@ def test_spar_hoa2_enc_system( ref_encoder_path, ref_decoder_path, update_ref, + keep_files, ivas_br, - tag + tag, ): fs = '48' dtx = '0' @@ -263,7 +269,8 @@ def test_spar_hoa2_enc_system( bypass, agc, output_config, - update_ref + update_ref, + keep_files, ) @pytest.mark.create_ref @@ -278,8 +285,9 @@ def test_spar_hoa3_enc_system( ref_encoder_path, ref_decoder_path, update_ref, + keep_files, ivas_br, - tag + tag, ): fs = '48' dtx = '0' @@ -324,7 +332,8 @@ def test_spar_hoa3_enc_system( bypass, agc, output_config, - update_ref + update_ref, + keep_files, ) @pytest.mark.create_ref @@ -341,10 +350,11 @@ def test_sba_enc_BWforce_system( ref_encoder_path, ref_decoder_path, update_ref, + keep_files, ivas_br, dtx, tag, - sample_rate_bw_idx + sample_rate_bw_idx, ): fs = sample_rate_bw_idx[0] bw = sample_rate_bw_idx[1] @@ -386,7 +396,8 @@ def test_sba_enc_BWforce_system( bypass, agc, output_config, - update_ref + update_ref, + keep_files, ) @@ -515,7 +526,7 @@ def sba_dec( agc, output_config, update_ref, - keep_files=False + keep_files, ): ######### run cmd ##################################### @@ -581,10 +592,10 @@ def sba_dec( end_skip_samples ) - ##File removal## + # report compare result + assert cmp_result == 0, reason + + # remove DUT output files when test result is OK (to save disk space) if not keep_files: os.remove(dut_in_pkt) os.remove(dut_out_raw) - - ##report failure - assert cmp_result == 0, reason -- GitLab From 48cd18779e99edd3be394932b7f931ae522bf4ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Thu, 29 Sep 2022 20:32:34 +0200 Subject: [PATCH 23/25] pytest: remove leading whitespace in copyright notice to reduce the number of pylint warnings --- tests/cmp_custom.py | 58 +++++++++++++++---------------- tests/conftest.py | 55 +++++++++++++++-------------- tests/create_short_testvectors.py | 56 ++++++++++++++--------------- tests/cut_pcm.py | 6 ++-- tests/run_pytests.py | 56 +++++++++++++++-------------- tests/test_param_file.py | 27 ++++++++------ tests/test_sba_bs_dec_plc.py | 55 +++++++++++++++-------------- tests/test_sba_bs_enc.py | 55 +++++++++++++++-------------- tests/testconfig.py | 6 ++-- 9 files changed, 194 insertions(+), 180 deletions(-) diff --git a/tests/cmp_custom.py b/tests/cmp_custom.py index 408d6408cd..ab22bc0ceb 100755 --- a/tests/cmp_custom.py +++ b/tests/cmp_custom.py @@ -1,34 +1,34 @@ #!/usr/bin/env python3 -__license__ = \ +__copyright__ = \ """ - (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ __doc__ = \ @@ -193,7 +193,7 @@ def cmp_custom( def main(argv) -> int: if len(argv) < 5: return usage() - retval, reason = cmp_custom(*argv[1:]) + retval, _reason = cmp_custom(*argv[1:]) return retval diff --git a/tests/conftest.py b/tests/conftest.py index 1decc4c7f7..da1c26e54a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,31 +1,32 @@ +__copyright__ = \ """ - (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ __doc__ = \ diff --git a/tests/create_short_testvectors.py b/tests/create_short_testvectors.py index 8cff1566b3..13afe8a422 100755 --- a/tests/create_short_testvectors.py +++ b/tests/create_short_testvectors.py @@ -1,34 +1,34 @@ #!/usr/bin/env python3 -__license__ = \ +__copyright__ = \ """ - (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ __doc__ = \ diff --git a/tests/cut_pcm.py b/tests/cut_pcm.py index 62af257a73..938cb6fc43 100755 --- a/tests/cut_pcm.py +++ b/tests/cut_pcm.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -__license__ = """ +__copyright__ = \ +""" (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, @@ -30,7 +31,8 @@ accordance with the laws of the Federal Republic of Germany excluding its confli the United Nations Convention on Contracts on the International Sales of Goods. """ -__doc__ = """ +__doc__ = \ +""" Script to cut samples from a 16-bit PCM file. USAGE : cut_pcm.py in_file_pcm out_file_pcm num_channels sample_rate start duration [gain] diff --git a/tests/run_pytests.py b/tests/run_pytests.py index cd94d27a1f..b23be3fa14 100755 --- a/tests/run_pytests.py +++ b/tests/run_pytests.py @@ -1,35 +1,37 @@ #!/usr/bin/env python3 +__copyright__ = \ """ - (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ +__doc__ = \ """ Script to run the pytest tests. diff --git a/tests/test_param_file.py b/tests/test_param_file.py index f690c67fec..730acd5c00 100644 --- a/tests/test_param_file.py +++ b/tests/test_param_file.py @@ -1,22 +1,27 @@ -__copyright__ = """ -(C) 2022 Baseline Development Group with portions copyright Dolby International AB, Ericsson AB, +__copyright__ = \ +""" +(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., -Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies OY, Orange, -Panasonic Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation. All Rights Reserved. +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. This software is protected by copyright law and by international treaties. -The Baseline Development Group consisting of Dolby International AB, Ericsson AB, +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., -Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies OY, Orange, -Panasonic Corporation, Qualcomm Technologies, Inc., and VoiceAge Corporation retain full ownership -rights in their respective contributions in the software. No license of any kind, including but not -limited to patent license, of any foregoing parties is hereby granted by implication, estoppel or -otherwise. +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. This software is provided "AS IS", without any express or implied warranties. The software is in the development stage. It is intended exclusively for experts who have experience with such software and solely for the purpose of inspection. All implied warranties of non-infringement, merchantability -and/or fitness for a particular purpose are hereby disclaimed and excluded. +and fitness for a particular purpose are hereby disclaimed and excluded. Any dispute, controversy or claim arising under or in relation to providing this software shall be submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in diff --git a/tests/test_sba_bs_dec_plc.py b/tests/test_sba_bs_dec_plc.py index 3922eed840..58be07ec33 100644 --- a/tests/test_sba_bs_dec_plc.py +++ b/tests/test_sba_bs_dec_plc.py @@ -1,31 +1,32 @@ +__copyright__ = \ """ - (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ __doc__ = \ diff --git a/tests/test_sba_bs_enc.py b/tests/test_sba_bs_enc.py index 048a7e46ab..3999164020 100644 --- a/tests/test_sba_bs_enc.py +++ b/tests/test_sba_bs_enc.py @@ -1,31 +1,32 @@ +__copyright__ = \ """ - (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ __doc__ = \ diff --git a/tests/testconfig.py b/tests/testconfig.py index 9837764910..f4827a004f 100644 --- a/tests/testconfig.py +++ b/tests/testconfig.py @@ -1,4 +1,5 @@ -__license__ = """ +__copyright__ = \ +""" (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, @@ -28,7 +29,8 @@ accordance with the laws of the Federal Republic of Germany excluding its confli the United Nations Convention on Contracts on the International Sales of Goods. """ -__doc__ = """ +__doc__ = \ +""" To configure test modules. """ -- GitLab From 688b860cfae2e130c589236086de584731d03613 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Fri, 30 Sep 2022 11:47:25 +0200 Subject: [PATCH 24/25] pytest: add prepare_pytests.py to complement modified run_pytests.py for local development --- tests/README.md | 41 ++++++++++++- tests/prepare_pytests.py | 123 +++++++++++++++++++++++++++++++++++++++ tests/run_pytests.py | 60 +++++-------------- 3 files changed, 178 insertions(+), 46 deletions(-) create mode 100755 tests/prepare_pytests.py diff --git a/tests/README.md b/tests/README.md index 470542760d..b87587798d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -50,7 +50,7 @@ The tests rely on references which need to be generated upfront using reference When the reference binaries are named `IVAS_cod_ref(.exe)` and `IVAS_dec_ref(.exe)`, pytest will find and use them. When the reference binaries are named differently, you need to specify them via the `--ref_encoder_path` and `--ref_decoder_path` options. -The tests will used the binaries `IVAS_cod(.exe)` and `IVAS_dec(.exe)` for testing. Please make sure that the binaries have been built before running the tests. +The tests will use the binaries `IVAS_cod(.exe)` and `IVAS_dec(.exe)` for testing. Please make sure that the binaries have been built before running the tests. When different test binaries are to be used, they can be specified via the `--dut_encoder_path` and `--dut_decoder_path` options. (DUT: Device Under Test) @@ -129,3 +129,42 @@ The custom options are listed as part of the pytest help `pytest -h`. --keep_files By default, the DUT output files of successful tests are deleted. Use --keep_files to prevent these deletions. ``` + +## Helper scripts + +To help with running the tests during development, two scripts are available in the `tests` folder: + +- prepare_pytests.py +- run_pytests.py + +The envisioned development workflow is: + +```bash +# 1. create a new git branch and switch to the branch +git checkout -b new_branch + +# 2. build the REF binaries (here: example for Linux) +make -j + +# 3. use the binaries to generate the references for future tests +# assumption: you want to test your development against the start of the development +tests/prepare_pytests.py +# Note: the script will use the binaries IVAS_cod and IVAS_dec in case IVAS_cod_ref and IVAS_dec_ref are not present + +# 3a. (optional) store REF binaries in case you want to re-run the reference generation at a later stage +cp IVAS_cod IVAS_cod_ref +cp IVAS_dec IVAS_dec_ref + +# 4. do the development changes +edit ... + +# 5. build the DUT binaries (here: example for Linux) +make -j + +# 6. run the tests +tests/run_pytests.py + +# 7. depending on the test result +# - either go back to 4. +# - or commit and push your changes +``` diff --git a/tests/prepare_pytests.py b/tests/prepare_pytests.py new file mode 100755 index 0000000000..dec9d63fa9 --- /dev/null +++ b/tests/prepare_pytests.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 + +__copyright__ = """ +(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +__doc__ = """ +Script to prepare the pytest tests. +""" + +import os +import sys +import argparse +import subprocess +import platform + +from pathlib import Path +from create_short_testvectors import create_short_testvectors + +BIN_EXT = ".exe" if platform.system() == "Windows" else "" +HERE = Path(__file__).parent.resolve() +DEFAULT_ENCODER_DUT = str(HERE.joinpath(f"../IVAS_cod{BIN_EXT}").resolve()) +DEFAULT_DECODER_DUT = str(HERE.joinpath(f"../IVAS_dec{BIN_EXT}").resolve()) +DEFAULT_ENCODER_REF = str(HERE.joinpath(f"../IVAS_cod_ref{BIN_EXT}").resolve()) +DEFAULT_DECODER_REF = str(HERE.joinpath(f"../IVAS_dec_ref{BIN_EXT}").resolve()) +REFERENCE_DIR = str(HERE.joinpath("ref").resolve()) + + +def main(argv): + """ + Prepare the pytest tests. + """ + # check for python >= 3.7 + if sys.version_info[0] < 3 or sys.version_info[1] < 7: + sys.exit("This script is written for Python >= 3.7. Found: " + platform.python_version()) + + parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + "--numprocesses", + action="store", + default="auto", + help="Number of processes to use in pytest (default: auto)", + ) + + args = parser.parse_args(argv[1:]) + + use_dut_binaries = False + + # check for existing references + if os.path.exists(REFERENCE_DIR): + sys.exit( + f"Found existing references directory {REFERENCE_DIR}.\n" + "Please delete this directory if you want the references to be recreated." + ) + + # check for DUT binaries + if not os.path.exists(DEFAULT_ENCODER_DUT) or not os.path.exists(DEFAULT_DECODER_DUT): + sys.exit( + f"Need DUT binaries {DEFAULT_ENCODER_DUT} and {DEFAULT_DECODER_DUT}.\n" + "Please create the binaries." + ) + + # check for REF binaries + if not os.path.exists(DEFAULT_ENCODER_REF) or not os.path.exists(DEFAULT_DECODER_REF): + print(f"REF binaries {DEFAULT_ENCODER_REF} and {DEFAULT_DECODER_REF} not found.") + print("DUT binaries will be used for reference generation.") + use_dut_binaries = True + + # create references + print(f"Creating references within the references directory {REFERENCE_DIR}.") + create_short_testvectors() + if platform.system() == "Windows": + base_cmd = ["pytest"] + else: + base_cmd = ["python3", "-m", "pytest"] + base_cmd += [ + "tests", + "-n", + args.numprocesses, + "--update_ref", + "1", + ] + if use_dut_binaries: + base_cmd += [ + "--ref_encoder_path", + DEFAULT_ENCODER_DUT, + "--ref_decoder_path", + DEFAULT_DECODER_DUT, + ] + + result = subprocess.run(base_cmd + ["-m", "create_ref"], check=False) + result = subprocess.run(base_cmd + ["-m", "create_ref_part2"], check=False) + return result.returncode + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tests/run_pytests.py b/tests/run_pytests.py index b23be3fa14..4d693f3559 100755 --- a/tests/run_pytests.py +++ b/tests/run_pytests.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 -__copyright__ = \ -""" +__copyright__ = """ (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, @@ -31,8 +30,7 @@ accordance with the laws of the Federal Republic of Germany excluding its confli the United Nations Convention on Contracts on the International Sales of Goods. """ -__doc__ = \ -""" +__doc__ = """ Script to run the pytest tests. Test prerequisites are checked for and check failures are reported. @@ -46,30 +44,22 @@ import subprocess import platform from pathlib import Path -sys.path.append('tests/') -from create_short_testvectors import create_short_testvectors - BIN_EXT = ".exe" if platform.system() == "Windows" else "" HERE = Path(__file__).parent.resolve() DEFAULT_ENCODER_DUT = str(HERE.joinpath(f"../IVAS_cod{BIN_EXT}").resolve()) DEFAULT_DECODER_DUT = str(HERE.joinpath(f"../IVAS_dec{BIN_EXT}").resolve()) -DEFAULT_ENCODER_REF = str(HERE.joinpath(f"../IVAS_cod_ref{BIN_EXT}").resolve()) -DEFAULT_DECODER_REF = str(HERE.joinpath(f"../IVAS_dec_ref{BIN_EXT}").resolve()) REFERENCE_DIR = str(HERE.joinpath("ref").resolve()) def main(argv): + """ + Run the pytest tests. + """ # check for python >= 3.7 if sys.version_info[0] < 3 or sys.version_info[1] < 7: sys.exit("This script is written for Python >= 3.7. Found: " + platform.python_version()) parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument( - "--create_only", - action="store_true", - default=False, - help="Create references when needed, but don't run the tests" - ) parser.add_argument( "--numprocesses", action="store", @@ -79,38 +69,18 @@ def main(argv): args = parser.parse_args(argv[1:]) + # check for references + if not os.path.exists(REFERENCE_DIR): + sys.exit( + f"References directory {REFERENCE_DIR} not found.\n" "Please create the references." + ) + # check for DUT binaries if not os.path.exists(DEFAULT_ENCODER_DUT) or not os.path.exists(DEFAULT_DECODER_DUT): - sys.exit(f"Need DUT binaries {DEFAULT_ENCODER_DUT} and {DEFAULT_DECODER_DUT}. Please create the binaries.") - - # check for references - if os.path.exists(REFERENCE_DIR): - print(f"Using existing references directory {REFERENCE_DIR}") - else: - # check for REF binaries - print(f"References directory {REFERENCE_DIR} does not exist.") - if not os.path.exists(DEFAULT_ENCODER_REF) or not os.path.exists(DEFAULT_DECODER_REF): - sys.exit(f"Need REF binaries {DEFAULT_ENCODER_REF} and {DEFAULT_DECODER_REF}. Please create the binaries.") - - # create references - print(f"Creating references within the references directory {REFERENCE_DIR}") - create_short_testvectors() - if platform.system() == "Windows": - base_cmd = ["pytest"] - else: - base_cmd = ["python3", "-m", "pytest"] - base_cmd += [ - "tests", - "-n", - args.numprocesses, - "--update_ref", - "1", - ] - subprocess.run(base_cmd + ["-m", "create_ref"], check=False) - subprocess.run(base_cmd + ["-m", "create_ref_part2"], check=False) - - if args.create_only: - return + sys.exit( + f"Need DUT binaries {DEFAULT_ENCODER_DUT} and {DEFAULT_DECODER_DUT}.\n" + "Please create the binaries." + ) # run pytest if platform.system() == "Windows": -- GitLab From 811bb182562faefba4e19b866272cb105e4bb033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhold=20B=C3=B6hm?= Date: Fri, 30 Sep 2022 16:20:13 +0200 Subject: [PATCH 25/25] pytest: add --param_file option to prepare_pytests.py and run_pytests.py --- tests/README.md | 3 +++ tests/prepare_pytests.py | 13 +++++++++++-- tests/run_pytests.py | 18 ++++++++++++------ 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/tests/README.md b/tests/README.md index b87587798d..6ba72e4ace 100644 --- a/tests/README.md +++ b/tests/README.md @@ -168,3 +168,6 @@ tests/run_pytests.py # - either go back to 4. # - or commit and push your changes ``` + +Both scripts allow to restrict the reference generation or the testing to test_param_file tests +with a custom `.prm` file via the `--param_file` option. diff --git a/tests/prepare_pytests.py b/tests/prepare_pytests.py index dec9d63fa9..d1f7495f07 100755 --- a/tests/prepare_pytests.py +++ b/tests/prepare_pytests.py @@ -67,6 +67,11 @@ def main(argv): default="auto", help="Number of processes to use in pytest (default: auto)", ) + parser.add_argument( + "--param_file", + action="store", + help="Restrict reference generation to test_param_file with specified param file.", + ) args = parser.parse_args(argv[1:]) @@ -99,8 +104,11 @@ def main(argv): base_cmd = ["pytest"] else: base_cmd = ["python3", "-m", "pytest"] + if args.param_file: + base_cmd += ["tests/test_param_file.py", "--param_file", args.param_file] + else: + base_cmd += ["tests"] base_cmd += [ - "tests", "-n", args.numprocesses, "--update_ref", @@ -115,7 +123,8 @@ def main(argv): ] result = subprocess.run(base_cmd + ["-m", "create_ref"], check=False) - result = subprocess.run(base_cmd + ["-m", "create_ref_part2"], check=False) + if not args.param_file: + result = subprocess.run(base_cmd + ["-m", "create_ref_part2"], check=False) return result.returncode diff --git a/tests/run_pytests.py b/tests/run_pytests.py index 4d693f3559..1fc6614762 100755 --- a/tests/run_pytests.py +++ b/tests/run_pytests.py @@ -66,13 +66,18 @@ def main(argv): default="auto", help="Number of processes to use in pytest (default: auto)", ) + parser.add_argument( + "--param_file", + action="store", + help="Restrict test run to test_param_file with specified param file.", + ) args = parser.parse_args(argv[1:]) # check for references if not os.path.exists(REFERENCE_DIR): sys.exit( - f"References directory {REFERENCE_DIR} not found.\n" "Please create the references." + f"References directory {REFERENCE_DIR} not found.\nPlease create the references." ) # check for DUT binaries @@ -87,11 +92,12 @@ def main(argv): cmd = ["pytest"] else: cmd = ["python3", "-m", "pytest"] - cmd += [ - "tests", - "-n", - args.numprocesses, - ] + if args.param_file: + cmd += ["tests/test_param_file.py", "--param_file", args.param_file] + else: + cmd += ["tests"] + cmd += ["-n", args.numprocesses] + result = subprocess.run(cmd, check=False) return result.returncode -- GitLab