From c07f6ee2e02b940ddbe812ba3b7a209068c424ca Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 3 Jul 2023 14:24:29 +0200 Subject: [PATCH 01/20] move MR pipeline BE tests into sudirectory --- .gitlab-ci.yml | 21 ++++++++++--------- tests/__init__.py | 0 tests/codec_be_on_mr/__init__.py | 0 .../{ => codec_be_on_mr}/test_masa_enc_dec.py | 4 ++-- tests/{ => codec_be_on_mr}/test_param_file.py | 8 +++---- .../test_sba_bs_dec_plc.py | 4 ++-- tests/{ => codec_be_on_mr}/test_sba_bs_enc.py | 6 +++--- tests/conftest.py | 2 +- tests/prepare_pytests.py | 6 +++--- tests/run_pytests.py | 4 ++-- 10 files changed, 28 insertions(+), 27 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/codec_be_on_mr/__init__.py rename tests/{ => codec_be_on_mr}/test_masa_enc_dec.py (99%) rename tests/{ => codec_be_on_mr}/test_param_file.py (98%) rename tests/{ => codec_be_on_mr}/test_sba_bs_dec_plc.py (98%) rename tests/{ => codec_be_on_mr}/test_sba_bs_enc.py (99%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b365c0e22d..b0fa618ac7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,6 +11,7 @@ variables: EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 PROCESSING_SCRIPTS_BIN_DIR: "/test-bin" + TESTS_DIR_CODEC_BE_ON_MR: "tests/codec_be_on_mr" default: interruptible: true # Make all jobs by default interruptible @@ -531,15 +532,15 @@ ivas-pytest-on-merge-request: # create short test vectors - python3 tests/create_short_testvectors.py # create references - - python3 -m pytest tests -v --update_ref 1 -m create_ref - - python3 -m pytest tests -v --update_ref 1 -m create_ref_part2 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2 ### Run test using branch scripts and input - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi ### run pytest - exit_code=0 - - python3 -m pytest tests -v --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check @@ -579,14 +580,14 @@ evs-pytest-on-merge-request: ### prepare pytest # create references - - python3 -m pytest tests/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm ### Run test using branch scripts and input - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi ### run pytest for EVS cases - exit_code=0 - - python3 -m pytest tests/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --html=report.html --self-contained-html --junit-xml=report-junit-evs.xml || exit_code=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --html=report.html --self-contained-html --junit-xml=report-junit-evs.xml || exit_code=$? - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check @@ -840,15 +841,15 @@ codec-comparison-on-main-push: - mv IVAS_cod_test IVAS_cod - mv IVAS_dec_test IVAS_dec # create references - - python3 -m pytest tests -v --update_ref 1 -m create_ref - - python3 -m pytest tests -v --update_ref 1 -m create_ref_part2 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2 ### re-checkout the latest commit here, if ref_using_main is set - if [ $ref_using_main -eq 1 ]; then git checkout $latest_commit;fi ### run pytest - exit_code=0 - - python3 -m pytest tests -v --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code=$? - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi @@ -1125,8 +1126,8 @@ coverage-test-on-main-scheduled: - make GCOV=1 -j - cp IVAS_rend IVAS_rend_ref # Copy exec to be able to run renderer script - python3 tests/create_short_testvectors.py - - python3 -m pytest tests -v -n 0 --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - - python3 -m pytest tests -v -n 0 --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n 0 --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n 0 --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - python3 -m pytest tests/test_param_file.py -v -n 0 --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - bash ci/smoke_test.sh coverage - python3 -m pytest -q -n auto tests/renderer/test_renderer_be_comparison.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/codec_be_on_mr/__init__.py b/tests/codec_be_on_mr/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_masa_enc_dec.py b/tests/codec_be_on_mr/test_masa_enc_dec.py similarity index 99% rename from tests/test_masa_enc_dec.py rename to tests/codec_be_on_mr/test_masa_enc_dec.py index bfa24c6d30..134611ae04 100644 --- a/tests/test_masa_enc_dec.py +++ b/tests/codec_be_on_mr/test_masa_enc_dec.py @@ -41,8 +41,8 @@ import pytest from typing import Optional from filecmp import cmp -from cmp_pcm import cmp_pcm -from conftest import EncoderFrontend, DecoderFrontend +from tests.conftest import EncoderFrontend, DecoderFrontend +from tests.cmp_pcm import cmp_pcm # params #output_mode_list = ['MONO', 'STEREO', '5_1', '7_1', '5_1_2', '5_1_4', '7_1_4', 'FOA', 'HOA2', 'HOA3', 'BINAURAL', 'BINAURAL_ROOM', 'EXT'] diff --git a/tests/test_param_file.py b/tests/codec_be_on_mr/test_param_file.py similarity index 98% rename from tests/test_param_file.py rename to tests/codec_be_on_mr/test_param_file.py index 5a2c64f589..02e00336f0 100644 --- a/tests/test_param_file.py +++ b/tests/codec_be_on_mr/test_param_file.py @@ -40,10 +40,10 @@ import platform import filecmp from subprocess import run import pytest -from cmp_pcm import cmp_pcm -from cut_pcm import cut_samples -from conftest import EncoderFrontend, DecoderFrontend -from testconfig import PARAM_FILE +from tests.cmp_pcm import cmp_pcm +from tests.cut_pcm import cut_samples +from tests.conftest import EncoderFrontend, DecoderFrontend +from tests.testconfig import PARAM_FILE VALID_DEC_OUTPUT_CONF = [ diff --git a/tests/test_sba_bs_dec_plc.py b/tests/codec_be_on_mr/test_sba_bs_dec_plc.py similarity index 98% rename from tests/test_sba_bs_dec_plc.py rename to tests/codec_be_on_mr/test_sba_bs_dec_plc.py index bead7cda93..c095f27558 100644 --- a/tests/test_sba_bs_dec_plc.py +++ b/tests/codec_be_on_mr/test_sba_bs_dec_plc.py @@ -38,8 +38,8 @@ import os import errno import pytest -from cmp_pcm import cmp_pcm -from conftest import DecoderFrontend +from tests.cmp_pcm import cmp_pcm +from tests.conftest import DecoderFrontend # params tag_list = ['stvFOA'] diff --git a/tests/test_sba_bs_enc.py b/tests/codec_be_on_mr/test_sba_bs_enc.py similarity index 99% rename from tests/test_sba_bs_enc.py rename to tests/codec_be_on_mr/test_sba_bs_enc.py index 759e1cce4f..3b05399784 100644 --- a/tests/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr/test_sba_bs_enc.py @@ -39,9 +39,9 @@ import os import errno import pytest -from cmp_pcm import cmp_pcm -from cut_pcm import cut_samples -from conftest import EncoderFrontend, DecoderFrontend +from tests.cmp_pcm import cmp_pcm +from tests.cut_pcm import cut_samples +from tests.conftest import EncoderFrontend, DecoderFrontend from cut_bs import cut_from_start # params diff --git a/tests/conftest.py b/tests/conftest.py index d7a35a36e0..958e0b10ac 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,7 +41,7 @@ from subprocess import run import textwrap from typing import Optional import os -import testconfig +import tests.testconfig import pytest logger = logging.getLogger(__name__) diff --git a/tests/prepare_pytests.py b/tests/prepare_pytests.py index 9074a6704a..69e1e25fea 100755 --- a/tests/prepare_pytests.py +++ b/tests/prepare_pytests.py @@ -98,12 +98,12 @@ def main(argv): else: base_cmd = ["python3", "-m", "pytest"] if args.param_file: - base_cmd += ["tests/test_param_file.py", "--param_file", args.param_file] + base_cmd += ["tests/codec_be_on_mr/test_param_file.py", "--param_file", args.param_file] else: - base_cmd += ["tests"] + base_cmd += ["tests/codec_be_on_mr"] base_cmd += [ "-n", - args.numprocesses, + "14", "--update_ref", "1", ] diff --git a/tests/run_pytests.py b/tests/run_pytests.py index d20de6ef72..368d599647 100755 --- a/tests/run_pytests.py +++ b/tests/run_pytests.py @@ -93,9 +93,9 @@ def main(argv): else: cmd = ["python3", "-m", "pytest"] if args.param_file: - cmd += ["tests/test_param_file.py", "--param_file", args.param_file] + cmd += ["tests/codec_be_on_mr/test_param_file.py", "--param_file", args.param_file] else: - cmd += ["tests"] + cmd += ["tests/codec_be_on_mr"] cmd += ["-n", args.numprocesses] result = subprocess.run(cmd, check=False) -- GitLab From e574247963d17f40e6543b5ee0ca58c56f6f887a Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 3 Jul 2023 15:17:03 +0200 Subject: [PATCH 02/20] rename things --- .gitlab-ci.yml | 2 +- .../__init__.py | 0 .../test_masa_enc_dec.py | 0 .../test_param_file.py | 0 .../test_sba_bs_dec_plc.py | 0 .../test_sba_bs_enc.py | 0 tests/prepare_pytests.py | 4 ++-- tests/run_pytests.py | 4 ++-- 8 files changed, 5 insertions(+), 5 deletions(-) rename tests/{codec_be_on_mr => codec_be_on_mr_nonselection}/__init__.py (100%) rename tests/{codec_be_on_mr => codec_be_on_mr_nonselection}/test_masa_enc_dec.py (100%) rename tests/{codec_be_on_mr => codec_be_on_mr_nonselection}/test_param_file.py (100%) rename tests/{codec_be_on_mr => codec_be_on_mr_nonselection}/test_sba_bs_dec_plc.py (100%) rename tests/{codec_be_on_mr => codec_be_on_mr_nonselection}/test_sba_bs_enc.py (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b0fa618ac7..fe829be035 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,7 +11,7 @@ variables: EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 PROCESSING_SCRIPTS_BIN_DIR: "/test-bin" - TESTS_DIR_CODEC_BE_ON_MR: "tests/codec_be_on_mr" + TESTS_DIR_CODEC_BE_ON_MR: "tests/codec_be_on_mr_nonselection" default: interruptible: true # Make all jobs by default interruptible diff --git a/tests/codec_be_on_mr/__init__.py b/tests/codec_be_on_mr_nonselection/__init__.py similarity index 100% rename from tests/codec_be_on_mr/__init__.py rename to tests/codec_be_on_mr_nonselection/__init__.py diff --git a/tests/codec_be_on_mr/test_masa_enc_dec.py b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py similarity index 100% rename from tests/codec_be_on_mr/test_masa_enc_dec.py rename to tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py diff --git a/tests/codec_be_on_mr/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py similarity index 100% rename from tests/codec_be_on_mr/test_param_file.py rename to tests/codec_be_on_mr_nonselection/test_param_file.py diff --git a/tests/codec_be_on_mr/test_sba_bs_dec_plc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py similarity index 100% rename from tests/codec_be_on_mr/test_sba_bs_dec_plc.py rename to tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py diff --git a/tests/codec_be_on_mr/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py similarity index 100% rename from tests/codec_be_on_mr/test_sba_bs_enc.py rename to tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py diff --git a/tests/prepare_pytests.py b/tests/prepare_pytests.py index 69e1e25fea..9e50ea90d1 100755 --- a/tests/prepare_pytests.py +++ b/tests/prepare_pytests.py @@ -98,9 +98,9 @@ def main(argv): else: base_cmd = ["python3", "-m", "pytest"] if args.param_file: - base_cmd += ["tests/codec_be_on_mr/test_param_file.py", "--param_file", args.param_file] + base_cmd += ["tests/codec_be_on_mr_nonselection/test_param_file.py", "--param_file", args.param_file] else: - base_cmd += ["tests/codec_be_on_mr"] + base_cmd += ["tests/codec_be_on_mr_nonselection"] base_cmd += [ "-n", "14", diff --git a/tests/run_pytests.py b/tests/run_pytests.py index 368d599647..91ab8f27d5 100755 --- a/tests/run_pytests.py +++ b/tests/run_pytests.py @@ -93,9 +93,9 @@ def main(argv): else: cmd = ["python3", "-m", "pytest"] if args.param_file: - cmd += ["tests/codec_be_on_mr/test_param_file.py", "--param_file", args.param_file] + cmd += ["tests/codec_be_on_mr_nonselection/test_param_file.py", "--param_file", args.param_file] else: - cmd += ["tests/codec_be_on_mr"] + cmd += ["tests/codec_be_on_mr_nonselection"] cmd += ["-n", args.numprocesses] result = subprocess.run(cmd, check=False) -- GitLab From 06809cf589deab9775fca587a61b002fa24dbfb8 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 4 Jul 2023 11:08:55 +0200 Subject: [PATCH 03/20] first version with single experiment OPs and limited parameters --- .gitignore | 7 ++-- tests/codec_be_on_mr_selection/__init__.py | 41 +++++++++++++++++++ tests/codec_be_on_mr_selection/constants.py | 8 ++++ .../test_experiments.py | 37 +++++++++++++++++ 4 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 tests/codec_be_on_mr_selection/__init__.py create mode 100644 tests/codec_be_on_mr_selection/constants.py create mode 100644 tests/codec_be_on_mr_selection/test_experiments.py diff --git a/.gitignore b/.gitignore index a2f6a6b95f..4d9dbe0474 100644 --- a/.gitignore +++ b/.gitignore @@ -42,10 +42,9 @@ scripts/out/ scripts/self_test_summary.txt scripts/cppp/ binary/ -tests/renderer/cut -tests/renderer/ref -tests/dut -tests/ref +tests/**/[c|d]ut +tests/**/ref +tests/*/testv scripts/testv/*_cut*.pcm # default reference binary name IVAS_cod_ref diff --git a/tests/codec_be_on_mr_selection/__init__.py b/tests/codec_be_on_mr_selection/__init__.py new file mode 100644 index 0000000000..87aba47645 --- /dev/null +++ b/tests/codec_be_on_mr_selection/__init__.py @@ -0,0 +1,41 @@ +import tempfile +import filecmp +from pathlib import Path +import subprocess +from .constants import TESTV_PATH, REF_PATH, DUT_PATH + + +def get_testvector_for_exp_cat_and_testset(experiment, category, testset): + fname = f"{experiment}-cat{category}-{testset}-input.wav" + return TESTV_PATH.joinpath(fname).absolute() + + +def get_out_bitstream_and_synthesis_name(input_signal, bitrate, dtx, fer): + dtx_str = "DTXon" if dtx else "DTXoff" + fer_str = "FER_5perc" if fer else "FER_0perc" + suffix = [".192", ".wav"] + return [DUT_PATH.joinpath(f"{input_signal.name}-{bitrate}bps-{dtx_str}-{fer_str}{s}") for s in suffix] + + +def get_error_pattern_for_exp_cat_and_testset(experiment, category, testset): + ep_file = TESTV_PATH.joinpath(f"{experiment}-cat{category}-{testset}-ep.192") + return ep_file + + +def apply_error_pattern_on_bitstream(in_bitstream: Path, error_pattern: Path, out_bitstream: Path): + # temporary file only really needed if same name is given for in and out bs + with tempfile.TemporaryDirectory() as tmpdir: + if in_bitstream == out_bitstream: + in_bitstream = Path(tmpdir).joinpath(in_bitstream.name) + + cmd = ["eid-xor", "-vbr", "-fer", in_bitstream, error_pattern, out_bitstream] + + subprocess.run(cmd) + + +def is_be_to_reference(dut_file: Path): + assert dut_file.parent == DUT_PATH + + # TODO: handle .wav files differently + ref_file = REF_PATH.joinpath(dut_file.name) + return filecmp.cmp(ref_file, dut_file) \ No newline at end of file diff --git a/tests/codec_be_on_mr_selection/constants.py b/tests/codec_be_on_mr_selection/constants.py new file mode 100644 index 0000000000..50209f7eb0 --- /dev/null +++ b/tests/codec_be_on_mr_selection/constants.py @@ -0,0 +1,8 @@ +from pathlib import Path + +HERE = Path(__file__).parent +TESTV_PATH = HERE.joinpath("testv") +REF_PATH = HERE.joinpath("ref") +DUT_PATH = HERE.joinpath("dut") + +SAMPLING_RATE = 48 \ No newline at end of file diff --git a/tests/codec_be_on_mr_selection/test_experiments.py b/tests/codec_be_on_mr_selection/test_experiments.py new file mode 100644 index 0000000000..00d4830666 --- /dev/null +++ b/tests/codec_be_on_mr_selection/test_experiments.py @@ -0,0 +1,37 @@ +import pytest +from . import get_testvector_for_exp_cat_and_testset, get_out_bitstream_and_synthesis_name, apply_error_pattern_on_bitstream, get_error_pattern_for_exp_cat_and_testset, is_be_to_reference + +P800_1_BITRATES = [13200, 16400, 24400, 32000, 48000, 13200, 16400, 24400, 32000, 48000, 24400, 13200] +P800_1_DTX = 10 * [False] + 2 * [True] +P800_1_FER = 5 * [False] + 5 * [True] + [False, True] +# TODO: use correct range +P800_1_CATEGORIES = range(1, 3) +P800_1_TESTSETS = ["a", "d"] +P800_1_PARAMS = list(zip(P800_1_BITRATES, P800_1_DTX, P800_1_FER)) + + +@pytest.mark.create_ref +@pytest.mark.parametrize("bitrate,dtx,fer", P800_1_PARAMS) +@pytest.mark.parametrize("category", P800_1_CATEGORIES) +@pytest.mark.parametrize("testset", P800_1_TESTSETS) +def test_p800_1(bitrate, dtx, fer, category, testset, dut_encoder_frontend, dut_decoder_frontend, update_ref): + sampling_rate = 48 + output_mode = "STEREO" + options = ["-stereo"] + + testv = get_testvector_for_exp_cat_and_testset("P800-1", category, testset) + dut_bitstream, dut_output = get_out_bitstream_and_synthesis_name(testv, bitrate, dtx, fer) + dut_encoder_frontend.run(bitrate, sampling_rate, testv, dut_bitstream, dtx_mode=dtx, add_option_list=options) + + if update_ref != 1 and not is_be_to_reference(dut_bitstream): + pytest.fail("Bitstream file differs from reference") + + if fer: + error_pattern = get_error_pattern_for_exp_cat_and_testset("P800-1", category, testset) + apply_error_pattern_on_bitstream(dut_bitstream, error_pattern, dut_bitstream) + + dut_decoder_frontend.run(output_mode, sampling_rate, dut_bitstream, dut_output) + + # TODO: also compare metadata if present + if update_ref != 1 and not is_be_to_reference(dut_output): + pytest.fail("Decoder output differs from reference") \ No newline at end of file -- GitLab From 2969baca9227bdab454d4dd4d0f65a6a7f9b5ef0 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 4 Jul 2023 11:47:54 +0200 Subject: [PATCH 04/20] refactor --- .../test_experiments.py | 50 ++++++++++++------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/tests/codec_be_on_mr_selection/test_experiments.py b/tests/codec_be_on_mr_selection/test_experiments.py index 00d4830666..58e8bbd964 100644 --- a/tests/codec_be_on_mr_selection/test_experiments.py +++ b/tests/codec_be_on_mr_selection/test_experiments.py @@ -1,37 +1,53 @@ import pytest from . import get_testvector_for_exp_cat_and_testset, get_out_bitstream_and_synthesis_name, apply_error_pattern_on_bitstream, get_error_pattern_for_exp_cat_and_testset, is_be_to_reference +P800_CATEGORIES = range(1, 3) + P800_1_BITRATES = [13200, 16400, 24400, 32000, 48000, 13200, 16400, 24400, 32000, 48000, 24400, 13200] P800_1_DTX = 10 * [False] + 2 * [True] P800_1_FER = 5 * [False] + 5 * [True] + [False, True] # TODO: use correct range -P800_1_CATEGORIES = range(1, 3) P800_1_TESTSETS = ["a", "d"] -P800_1_PARAMS = list(zip(P800_1_BITRATES, P800_1_DTX, P800_1_FER)) +P800_1_PARAMS = zip(P800_1_BITRATES, P800_1_DTX, P800_1_FER) +P800_2_BITRATES = [13200, 16400, 24400, 32000, 48000, 64000, 13200, 16400, 24400, 32000, 48000] +P800_2_DTX = 6 * [False] + 5 * [False] +P800_2_TESTSETS = ["b", "d"] +P800_2_PARAMS = list(zip(P800_2_BITRATES, P800_2_DTX)) -@pytest.mark.create_ref -@pytest.mark.parametrize("bitrate,dtx,fer", P800_1_PARAMS) -@pytest.mark.parametrize("category", P800_1_CATEGORIES) -@pytest.mark.parametrize("testset", P800_1_TESTSETS) -def test_p800_1(bitrate, dtx, fer, category, testset, dut_encoder_frontend, dut_decoder_frontend, update_ref): +OUTPUT_MODES_AND_OPTIONS_FOR_EXPERIMENT = { + "P800-1": ("STEREO", ["-stereo"]), + "P800-2": ("STEREO", ["-stereo"]), +} + + +def run_check(experiment, category, testset, bitrate, dtx, fer, encoder_frontend, decoder_frontend, is_ref_creation): sampling_rate = 48 - output_mode = "STEREO" - options = ["-stereo"] + output_mode, options = OUTPUT_MODES_AND_OPTIONS_FOR_EXPERIMENT[experiment] + + testv = get_testvector_for_exp_cat_and_testset(experiment, category, testset) + error_pattern = get_error_pattern_for_exp_cat_and_testset(experiment, category, testset) if fer else None - testv = get_testvector_for_exp_cat_and_testset("P800-1", category, testset) dut_bitstream, dut_output = get_out_bitstream_and_synthesis_name(testv, bitrate, dtx, fer) - dut_encoder_frontend.run(bitrate, sampling_rate, testv, dut_bitstream, dtx_mode=dtx, add_option_list=options) + encoder_frontend.run(bitrate, sampling_rate, testv, dut_bitstream, dtx_mode=dtx, add_option_list=options) - if update_ref != 1 and not is_be_to_reference(dut_bitstream): + if not is_ref_creation and not is_be_to_reference(dut_bitstream): pytest.fail("Bitstream file differs from reference") - if fer: - error_pattern = get_error_pattern_for_exp_cat_and_testset("P800-1", category, testset) + if error_pattern is not None: apply_error_pattern_on_bitstream(dut_bitstream, error_pattern, dut_bitstream) - dut_decoder_frontend.run(output_mode, sampling_rate, dut_bitstream, dut_output) + decoder_frontend.run(output_mode, sampling_rate, dut_bitstream, dut_output) # TODO: also compare metadata if present - if update_ref != 1 and not is_be_to_reference(dut_output): - pytest.fail("Decoder output differs from reference") \ No newline at end of file + if not is_ref_creation and not is_be_to_reference(dut_output): + pytest.fail("Decoder output differs from reference") + + +@pytest.mark.create_ref +@pytest.mark.parametrize("bitrate,dtx,fer", P800_1_PARAMS) +@pytest.mark.parametrize("category", P800_CATEGORIES) +@pytest.mark.parametrize("testset", P800_1_TESTSETS) +def test_p800_1(bitrate, dtx, fer, category, testset, dut_encoder_frontend, dut_decoder_frontend, update_ref): + experiment = "P800-1" + run_check(experiment, category, testset, bitrate, dtx, fer, dut_encoder_frontend, dut_decoder_frontend, update_ref == 1) \ No newline at end of file -- GitLab From 2b3ca2eab3af2a58fd0db6b3ac8bc6fcad3b17c4 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 6 Jul 2023 10:32:15 +0200 Subject: [PATCH 05/20] unified into one parametrized testcase --- tests/codec_be_on_mr_selection/__init__.py | 54 ++++- tests/codec_be_on_mr_selection/constants.py | 206 +++++++++++++++++- .../test_experiments.py | 52 +---- 3 files changed, 255 insertions(+), 57 deletions(-) diff --git a/tests/codec_be_on_mr_selection/__init__.py b/tests/codec_be_on_mr_selection/__init__.py index 87aba47645..d7c4fc27e0 100644 --- a/tests/codec_be_on_mr_selection/__init__.py +++ b/tests/codec_be_on_mr_selection/__init__.py @@ -1,20 +1,40 @@ import tempfile +import pytest import filecmp from pathlib import Path import subprocess -from .constants import TESTV_PATH, REF_PATH, DUT_PATH +from .constants import OUTPUT_MODES_AND_OPTIONS_FOR_EXPERIMENT +HERE = Path(__file__).parent +TESTV_PATH = HERE.joinpath("testv") +REF_PATH = HERE.joinpath("ref") +DUT_PATH = HERE.joinpath("dut") -def get_testvector_for_exp_cat_and_testset(experiment, category, testset): +ISM_NUM_FOR_EXP = { + "P800-6": 1, + "P800-7": 2, + "BS1534-6a": 3, + "BS1534-6b": 4, +} +MASA_TESTS = ("P800-8", "P800-9", "BS1534-7a", "BS1534-7a") + +def get_testvectors_for_exp_cat_and_testset(experiment, category, testset): fname = f"{experiment}-cat{category}-{testset}-input.wav" - return TESTV_PATH.joinpath(fname).absolute() + signal = TESTV_PATH.joinpath(fname).absolute() + + # will be an empty list if not ISM experiment + metadata = sorted([TESTV_PATH.joinpath(fname + f"{i}.csv") for i in range(ISM_NUM_FOR_EXP.get(experiment, 0))]) + if experiment in MASA_TESTS: + metadata.append(TESTV_PATH.joinpath(fname + ".met")) + + return signal, metadata def get_out_bitstream_and_synthesis_name(input_signal, bitrate, dtx, fer): dtx_str = "DTXon" if dtx else "DTXoff" fer_str = "FER_5perc" if fer else "FER_0perc" suffix = [".192", ".wav"] - return [DUT_PATH.joinpath(f"{input_signal.name}-{bitrate}bps-{dtx_str}-{fer_str}{s}") for s in suffix] + return [DUT_PATH.joinpath(f"{input_signal.stem}-{bitrate}bps-{dtx_str}-{fer_str}{s}") for s in suffix] def get_error_pattern_for_exp_cat_and_testset(experiment, category, testset): @@ -38,4 +58,28 @@ def is_be_to_reference(dut_file: Path): # TODO: handle .wav files differently ref_file = REF_PATH.joinpath(dut_file.name) - return filecmp.cmp(ref_file, dut_file) \ No newline at end of file + return filecmp.cmp(ref_file, dut_file) + + +def run_check(experiment, category, testset, bitrate, dtx, fer, encoder_frontend, decoder_frontend, is_ref_creation): + sampling_rate = 48 + output_mode, options = OUTPUT_MODES_AND_OPTIONS_FOR_EXPERIMENT[experiment] + + testv, metadata = get_testvectors_for_exp_cat_and_testset(experiment, category, testset) + error_pattern = get_error_pattern_for_exp_cat_and_testset(experiment, category, testset) if fer else None + dut_bitstream, dut_output = get_out_bitstream_and_synthesis_name(testv, bitrate, dtx, fer) + + options.extend(metadata) + encoder_frontend.run(bitrate, sampling_rate, testv, dut_bitstream, dtx_mode=dtx, add_option_list=options) + + if not is_ref_creation and not is_be_to_reference(dut_bitstream): + pytest.fail("Bitstream file differs from reference") + + if error_pattern is not None: + apply_error_pattern_on_bitstream(dut_bitstream, error_pattern, dut_bitstream) + + decoder_frontend.run(output_mode, sampling_rate, dut_bitstream, dut_output) + + # TODO: also compare metadata if present + if not is_ref_creation and not is_be_to_reference(dut_output): + pytest.fail("Decoder output differs from reference") \ No newline at end of file diff --git a/tests/codec_be_on_mr_selection/constants.py b/tests/codec_be_on_mr_selection/constants.py index 50209f7eb0..010a56a757 100644 --- a/tests/codec_be_on_mr_selection/constants.py +++ b/tests/codec_be_on_mr_selection/constants.py @@ -1,8 +1,202 @@ -from pathlib import Path +from itertools import product +from typing import Union, List -HERE = Path(__file__).parent -TESTV_PATH = HERE.joinpath("testv") -REF_PATH = HERE.joinpath("ref") -DUT_PATH = HERE.joinpath("dut") +def zip_with_check(*args): + """ + Like normal zip, but raises error if given iterables have different lengths. + Used here to avoid forgetting a bitrate or so. + """ + lengths = [len(it) for it in args] + assert len(set(lengths)) == 1 -SAMPLING_RATE = 48 \ No newline at end of file + return zip(*args) + +def assemble_unified_params_for_experiment(experiment, params, testset): + return [(experiment, *p, t) for p, t in product(params, testset)] + + +class ExperimentParams(): + def __init__(self, name: str, n_conditions: int, bitrates: List[int], testsets: List[str], dtx: Union[List[bool], bool] = False, fer: Union[List[bool], bool] = False): + dtx = n_conditions * [dtx] if isinstance(dtx, bool) else dtx + fer = n_conditions * [fer] if isinstance(fer, bool) else fer + assert len(bitrates) == n_conditions and len(testsets) == 2 and len(dtx) == n_conditions and len(fer) == n_conditions + + self.name = name + self.bitrates = bitrates + self.testsets = testsets + self.dtx = dtx + self.fer = fer + + def assemble_unified_params(self): + params = zip(self.bitrates, self.dtx, self.fer) + return [(self.name, *p, t) for p, t in product(params, self.testsets)] + + +P800_CATEGORIES = range(1, 7) + +EXPERIMENT_PARAMS = [ + ExperimentParams( + name="P800-1", + n_conditions=12, + bitrates=[13200, 16400, 24400, 32000, 48000, 13200, 16400, 24400, 32000, 48000, 24400, 13200], + dtx=10 * [False] + 2 * [True], + fer=5 * [False] + 5 * [True] + [False, True], + testsets=["a", "d"] + ), + ExperimentParams( + name="P800-2", + n_conditions=11, + bitrates=[13200, 16400, 24400, 32000, 48000, 64000, 13200, 16400, 24400, 32000, 48000], + dtx=6 * [False] + 5 * [True], + testsets=["b", "d"] + ), + ExperimentParams( + name="P800-3", + n_conditions=13, + bitrates=[13200, 16400, 24400, 32000, 48000, 64000, 13200, 16400, 24400, 32000, 48000, 24400, 13200], + dtx=11 * [False] + 2 * [True], + fer=6 * [False] + 5 * [True] + [False, True], + testsets=["a", "d"] + ), + ExperimentParams( + name="P800-4", + n_conditions=13, + bitrates=[16400, 24400, 32000, 48000, 64000, 80000, 96000, 24400, 32000, 48000, 64000, 80000, 96000], + fer=7 * [False] + 6 * [True], + testsets=["a", "c"] + ), + ExperimentParams( + name="P800-5", + n_conditions=13, + bitrates=[16400, 24400, 32000, 48000, 64000, 80000, 96000, 16400, 24400, 32000, 48000, 64000, 80000], + dtx=7 * [False] + 6 * [True], + testsets=["a", "b"] + ), + ExperimentParams( + name="P800-6", + n_conditions=13, + bitrates=[13200, 16400, 24400, 32000, 48000, 64000, 13200, 16400, 24400, 32000, 13200, 16400, 24400], + dtx=10 * [False] + 3 * [True], + fer=6 * [False] + 4 * [True] + 3 * [False], + testsets=["a", "c"] + ), + ExperimentParams( + name="P800-7", + n_conditions=13, + bitrates=[16400, 24400, 32000, 48000, 64000, 16400, 24400, 32000, 48000, 16400, 24400, 32000, 48000], + dtx=9 * [False] + 4 * [True], + fer=5 * [False] + 4 * [True] + 4 * [False], + testsets=["a", "d"] + ), + ExperimentParams( + name="P800-8", + n_conditions=12, + bitrates=[13200, 16400, 24400, 32000, 48000, 64000, 80000, 13200, 16400, 24400, 48000, 64000], + fer=7 * [False] + 5 * [True], + testsets=["a", "b"] + ), + ExperimentParams( + name="P800-9", + n_conditions=13, + bitrates=[13200, 16400, 24400, 32000, 48000, 64000, 80000, 13200, 16400, 24400, 32000, 48000, 64000], + dtx=7 * [False] + 6 * [True], + testsets=["a", "d"] + ), + + ExperimentParams( + name="BS1534-1a", + n_conditions=2, + bitrates=[48000, 64000], + testsets=["a", "d"] + ), + ExperimentParams( + name="BS1534-1b", + n_conditions=2, + bitrates=[96000, 128000], + testsets=["b", "d"] + ), + ExperimentParams( + name="BS1534-2a", + n_conditions=2, + bitrates=[64000, 96000], + testsets=["a", "b"] + ), + ExperimentParams( + name="BS1534-2b", + n_conditions=2, + bitrates=[128000, 160000], + testsets=["a", "b"] + ), + ExperimentParams( + name="BS1534-3a", + n_conditions=2, + bitrates=[128000, 160000], + testsets=["a", "b"] + ), + ExperimentParams( + name="BS1534-3b", + n_conditions=2, + bitrates=[384000, 512000], + testsets=["a", "b"] + ), + ExperimentParams( + name="BS1534-4a", + n_conditions=3, + bitrates=[96000, 128000, 160000], + testsets=["a", "d"] + ), + ExperimentParams( + name="BS1534-4b", + n_conditions=2, + bitrates=[160000, 192000], + testsets=["b", "d"] + ), + ExperimentParams( + name="BS1534-5a", + n_conditions=2, + bitrates=[192000, 256000], + testsets=["a", "d"] + ), + ExperimentParams( + name="BS1534-5b", + n_conditions=2, + bitrates=[384000, 512000], + testsets=["a", "b"] + ), + ExperimentParams( + name="BS1534-6a", + n_conditions=3, + bitrates=[48000, 64000, 96000], + testsets=["b", "d"] + ), + ExperimentParams( + name="BS1534-6b", + n_conditions=3, + bitrates=[96000, 128000, 256000], + testsets=["b", "d"] + ), + ExperimentParams( + name="BS1534-7a", + n_conditions=2, + bitrates=[96000, 128000], + testsets=["b", "d"] + ), + ExperimentParams( + name="BS1534-7b", + n_conditions=2, + bitrates=[192000, 256000], + testsets=["b", "d"] + ), +] + +OUTPUT_MODES_AND_OPTIONS_FOR_EXPERIMENT = { + "P800-1": ("STEREO", ["-stereo"]), + "P800-2": ("STEREO", ["-stereo"]), + "P800-3": ("STEREO", ["-stereo"]), + "P800-4": ("FOA", ["-sba", "1"]), + "P800-5": ("FOA", ["-sba", "1"]), + "P800-6": ("EXT", ["-ism", "1"]), + "P800-7": ("EXT", ["-ism", "2"]), + "P800-8": ("EXT", ["-masa", "2"]), + "P800-9": ("EXT", ["-masa", "2"]), +} diff --git a/tests/codec_be_on_mr_selection/test_experiments.py b/tests/codec_be_on_mr_selection/test_experiments.py index 58e8bbd964..5552d83e91 100644 --- a/tests/codec_be_on_mr_selection/test_experiments.py +++ b/tests/codec_be_on_mr_selection/test_experiments.py @@ -1,53 +1,13 @@ import pytest -from . import get_testvector_for_exp_cat_and_testset, get_out_bitstream_and_synthesis_name, apply_error_pattern_on_bitstream, get_error_pattern_for_exp_cat_and_testset, is_be_to_reference +from . import run_check +from .constants import EXPERIMENT_PARAMS, P800_CATEGORIES -P800_CATEGORIES = range(1, 3) -P800_1_BITRATES = [13200, 16400, 24400, 32000, 48000, 13200, 16400, 24400, 32000, 48000, 24400, 13200] -P800_1_DTX = 10 * [False] + 2 * [True] -P800_1_FER = 5 * [False] + 5 * [True] + [False, True] -# TODO: use correct range -P800_1_TESTSETS = ["a", "d"] -P800_1_PARAMS = zip(P800_1_BITRATES, P800_1_DTX, P800_1_FER) - -P800_2_BITRATES = [13200, 16400, 24400, 32000, 48000, 64000, 13200, 16400, 24400, 32000, 48000] -P800_2_DTX = 6 * [False] + 5 * [False] -P800_2_TESTSETS = ["b", "d"] -P800_2_PARAMS = list(zip(P800_2_BITRATES, P800_2_DTX)) - -OUTPUT_MODES_AND_OPTIONS_FOR_EXPERIMENT = { - "P800-1": ("STEREO", ["-stereo"]), - "P800-2": ("STEREO", ["-stereo"]), -} - - -def run_check(experiment, category, testset, bitrate, dtx, fer, encoder_frontend, decoder_frontend, is_ref_creation): - sampling_rate = 48 - output_mode, options = OUTPUT_MODES_AND_OPTIONS_FOR_EXPERIMENT[experiment] - - testv = get_testvector_for_exp_cat_and_testset(experiment, category, testset) - error_pattern = get_error_pattern_for_exp_cat_and_testset(experiment, category, testset) if fer else None - - dut_bitstream, dut_output = get_out_bitstream_and_synthesis_name(testv, bitrate, dtx, fer) - encoder_frontend.run(bitrate, sampling_rate, testv, dut_bitstream, dtx_mode=dtx, add_option_list=options) - - if not is_ref_creation and not is_be_to_reference(dut_bitstream): - pytest.fail("Bitstream file differs from reference") - - if error_pattern is not None: - apply_error_pattern_on_bitstream(dut_bitstream, error_pattern, dut_bitstream) - - decoder_frontend.run(output_mode, sampling_rate, dut_bitstream, dut_output) - - # TODO: also compare metadata if present - if not is_ref_creation and not is_be_to_reference(dut_output): - pytest.fail("Decoder output differs from reference") +P800_PARAMS_UNIFIED = [i for sl in [p.assemble_unified_params() for p in EXPERIMENT_PARAMS] for i in sl] @pytest.mark.create_ref -@pytest.mark.parametrize("bitrate,dtx,fer", P800_1_PARAMS) +@pytest.mark.parametrize("experiment,bitrate,dtx,fer,testset", P800_PARAMS_UNIFIED) @pytest.mark.parametrize("category", P800_CATEGORIES) -@pytest.mark.parametrize("testset", P800_1_TESTSETS) -def test_p800_1(bitrate, dtx, fer, category, testset, dut_encoder_frontend, dut_decoder_frontend, update_ref): - experiment = "P800-1" - run_check(experiment, category, testset, bitrate, dtx, fer, dut_encoder_frontend, dut_decoder_frontend, update_ref == 1) \ No newline at end of file +def test_experiment(experiment, bitrate, dtx, fer, testset, category, dut_encoder_frontend, dut_decoder_frontend, update_ref): + run_check(experiment, category, testset, bitrate, dtx, fer, dut_encoder_frontend, dut_decoder_frontend, update_ref == 1) -- GitLab From c6586b1a36163114505b128bd35f791c2f701941 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 6 Jul 2023 10:35:59 +0200 Subject: [PATCH 06/20] cleanup and formatting --- tests/codec_be_on_mr_selection/__init__.py | 56 +++- tests/codec_be_on_mr_selection/constants.py | 254 ++++++++++++------ .../test_experiments.py | 30 ++- 3 files changed, 243 insertions(+), 97 deletions(-) diff --git a/tests/codec_be_on_mr_selection/__init__.py b/tests/codec_be_on_mr_selection/__init__.py index d7c4fc27e0..714de01924 100644 --- a/tests/codec_be_on_mr_selection/__init__.py +++ b/tests/codec_be_on_mr_selection/__init__.py @@ -18,12 +18,18 @@ ISM_NUM_FOR_EXP = { } MASA_TESTS = ("P800-8", "P800-9", "BS1534-7a", "BS1534-7a") + def get_testvectors_for_exp_cat_and_testset(experiment, category, testset): fname = f"{experiment}-cat{category}-{testset}-input.wav" signal = TESTV_PATH.joinpath(fname).absolute() # will be an empty list if not ISM experiment - metadata = sorted([TESTV_PATH.joinpath(fname + f"{i}.csv") for i in range(ISM_NUM_FOR_EXP.get(experiment, 0))]) + metadata = sorted( + [ + TESTV_PATH.joinpath(fname + f"{i}.csv") + for i in range(ISM_NUM_FOR_EXP.get(experiment, 0)) + ] + ) if experiment in MASA_TESTS: metadata.append(TESTV_PATH.joinpath(fname + ".met")) @@ -34,7 +40,10 @@ def get_out_bitstream_and_synthesis_name(input_signal, bitrate, dtx, fer): dtx_str = "DTXon" if dtx else "DTXoff" fer_str = "FER_5perc" if fer else "FER_0perc" suffix = [".192", ".wav"] - return [DUT_PATH.joinpath(f"{input_signal.stem}-{bitrate}bps-{dtx_str}-{fer_str}{s}") for s in suffix] + return [ + DUT_PATH.joinpath(f"{input_signal.stem}-{bitrate}bps-{dtx_str}-{fer_str}{s}") + for s in suffix + ] def get_error_pattern_for_exp_cat_and_testset(experiment, category, testset): @@ -42,12 +51,14 @@ def get_error_pattern_for_exp_cat_and_testset(experiment, category, testset): return ep_file -def apply_error_pattern_on_bitstream(in_bitstream: Path, error_pattern: Path, out_bitstream: Path): +def apply_error_pattern_on_bitstream( + in_bitstream: Path, error_pattern: Path, out_bitstream: Path +): # temporary file only really needed if same name is given for in and out bs with tempfile.TemporaryDirectory() as tmpdir: if in_bitstream == out_bitstream: in_bitstream = Path(tmpdir).joinpath(in_bitstream.name) - + cmd = ["eid-xor", "-vbr", "-fer", in_bitstream, error_pattern, out_bitstream] subprocess.run(cmd) @@ -61,16 +72,41 @@ def is_be_to_reference(dut_file: Path): return filecmp.cmp(ref_file, dut_file) -def run_check(experiment, category, testset, bitrate, dtx, fer, encoder_frontend, decoder_frontend, is_ref_creation): +def run_check( + experiment, + category, + testset, + bitrate, + dtx, + fer, + encoder_frontend, + decoder_frontend, + is_ref_creation, +): sampling_rate = 48 output_mode, options = OUTPUT_MODES_AND_OPTIONS_FOR_EXPERIMENT[experiment] - testv, metadata = get_testvectors_for_exp_cat_and_testset(experiment, category, testset) - error_pattern = get_error_pattern_for_exp_cat_and_testset(experiment, category, testset) if fer else None - dut_bitstream, dut_output = get_out_bitstream_and_synthesis_name(testv, bitrate, dtx, fer) + testv, metadata = get_testvectors_for_exp_cat_and_testset( + experiment, category, testset + ) + error_pattern = ( + get_error_pattern_for_exp_cat_and_testset(experiment, category, testset) + if fer + else None + ) + dut_bitstream, dut_output = get_out_bitstream_and_synthesis_name( + testv, bitrate, dtx, fer + ) options.extend(metadata) - encoder_frontend.run(bitrate, sampling_rate, testv, dut_bitstream, dtx_mode=dtx, add_option_list=options) + encoder_frontend.run( + bitrate, + sampling_rate, + testv, + dut_bitstream, + dtx_mode=dtx, + add_option_list=options, + ) if not is_ref_creation and not is_be_to_reference(dut_bitstream): pytest.fail("Bitstream file differs from reference") @@ -82,4 +118,4 @@ def run_check(experiment, category, testset, bitrate, dtx, fer, encoder_frontend # TODO: also compare metadata if present if not is_ref_creation and not is_be_to_reference(dut_output): - pytest.fail("Decoder output differs from reference") \ No newline at end of file + pytest.fail("Decoder output differs from reference") diff --git a/tests/codec_be_on_mr_selection/constants.py b/tests/codec_be_on_mr_selection/constants.py index 010a56a757..f1fb94dbbb 100644 --- a/tests/codec_be_on_mr_selection/constants.py +++ b/tests/codec_be_on_mr_selection/constants.py @@ -1,25 +1,25 @@ from itertools import product from typing import Union, List -def zip_with_check(*args): - """ - Like normal zip, but raises error if given iterables have different lengths. - Used here to avoid forgetting a bitrate or so. - """ - lengths = [len(it) for it in args] - assert len(set(lengths)) == 1 - return zip(*args) - -def assemble_unified_params_for_experiment(experiment, params, testset): - return [(experiment, *p, t) for p, t in product(params, testset)] - - -class ExperimentParams(): - def __init__(self, name: str, n_conditions: int, bitrates: List[int], testsets: List[str], dtx: Union[List[bool], bool] = False, fer: Union[List[bool], bool] = False): +class ExperimentParams: + def __init__( + self, + name: str, + n_conditions: int, + bitrates: List[int], + testsets: List[str], + dtx: Union[List[bool], bool] = False, + fer: Union[List[bool], bool] = False, + ): dtx = n_conditions * [dtx] if isinstance(dtx, bool) else dtx fer = n_conditions * [fer] if isinstance(fer, bool) else fer - assert len(bitrates) == n_conditions and len(testsets) == 2 and len(dtx) == n_conditions and len(fer) == n_conditions + assert ( + len(bitrates) == n_conditions + and len(testsets) == 2 + and len(dtx) == n_conditions + and len(fer) == n_conditions + ) self.name = name self.bitrates = bitrates @@ -38,154 +38,242 @@ EXPERIMENT_PARAMS = [ ExperimentParams( name="P800-1", n_conditions=12, - bitrates=[13200, 16400, 24400, 32000, 48000, 13200, 16400, 24400, 32000, 48000, 24400, 13200], - dtx=10 * [False] + 2 * [True], + bitrates=[ + 13200, + 16400, + 24400, + 32000, + 48000, + 13200, + 16400, + 24400, + 32000, + 48000, + 24400, + 13200, + ], + dtx=10 * [False] + 2 * [True], fer=5 * [False] + 5 * [True] + [False, True], - testsets=["a", "d"] + testsets=["a", "d"], ), ExperimentParams( name="P800-2", n_conditions=11, - bitrates=[13200, 16400, 24400, 32000, 48000, 64000, 13200, 16400, 24400, 32000, 48000], + bitrates=[ + 13200, + 16400, + 24400, + 32000, + 48000, + 64000, + 13200, + 16400, + 24400, + 32000, + 48000, + ], dtx=6 * [False] + 5 * [True], - testsets=["b", "d"] + testsets=["b", "d"], ), ExperimentParams( name="P800-3", n_conditions=13, - bitrates=[13200, 16400, 24400, 32000, 48000, 64000, 13200, 16400, 24400, 32000, 48000, 24400, 13200], + bitrates=[ + 13200, + 16400, + 24400, + 32000, + 48000, + 64000, + 13200, + 16400, + 24400, + 32000, + 48000, + 24400, + 13200, + ], dtx=11 * [False] + 2 * [True], fer=6 * [False] + 5 * [True] + [False, True], - testsets=["a", "d"] + testsets=["a", "d"], ), ExperimentParams( name="P800-4", n_conditions=13, - bitrates=[16400, 24400, 32000, 48000, 64000, 80000, 96000, 24400, 32000, 48000, 64000, 80000, 96000], + bitrates=[ + 16400, + 24400, + 32000, + 48000, + 64000, + 80000, + 96000, + 24400, + 32000, + 48000, + 64000, + 80000, + 96000, + ], fer=7 * [False] + 6 * [True], - testsets=["a", "c"] + testsets=["a", "c"], ), ExperimentParams( name="P800-5", n_conditions=13, - bitrates=[16400, 24400, 32000, 48000, 64000, 80000, 96000, 16400, 24400, 32000, 48000, 64000, 80000], + bitrates=[ + 16400, + 24400, + 32000, + 48000, + 64000, + 80000, + 96000, + 16400, + 24400, + 32000, + 48000, + 64000, + 80000, + ], dtx=7 * [False] + 6 * [True], - testsets=["a", "b"] + testsets=["a", "b"], ), ExperimentParams( name="P800-6", n_conditions=13, - bitrates=[13200, 16400, 24400, 32000, 48000, 64000, 13200, 16400, 24400, 32000, 13200, 16400, 24400], + bitrates=[ + 13200, + 16400, + 24400, + 32000, + 48000, + 64000, + 13200, + 16400, + 24400, + 32000, + 13200, + 16400, + 24400, + ], dtx=10 * [False] + 3 * [True], fer=6 * [False] + 4 * [True] + 3 * [False], - testsets=["a", "c"] + testsets=["a", "c"], ), ExperimentParams( name="P800-7", n_conditions=13, - bitrates=[16400, 24400, 32000, 48000, 64000, 16400, 24400, 32000, 48000, 16400, 24400, 32000, 48000], + bitrates=[ + 16400, + 24400, + 32000, + 48000, + 64000, + 16400, + 24400, + 32000, + 48000, + 16400, + 24400, + 32000, + 48000, + ], dtx=9 * [False] + 4 * [True], fer=5 * [False] + 4 * [True] + 4 * [False], - testsets=["a", "d"] + testsets=["a", "d"], ), ExperimentParams( name="P800-8", n_conditions=12, - bitrates=[13200, 16400, 24400, 32000, 48000, 64000, 80000, 13200, 16400, 24400, 48000, 64000], + bitrates=[ + 13200, + 16400, + 24400, + 32000, + 48000, + 64000, + 80000, + 13200, + 16400, + 24400, + 48000, + 64000, + ], fer=7 * [False] + 5 * [True], - testsets=["a", "b"] + testsets=["a", "b"], ), ExperimentParams( name="P800-9", n_conditions=13, - bitrates=[13200, 16400, 24400, 32000, 48000, 64000, 80000, 13200, 16400, 24400, 32000, 48000, 64000], + bitrates=[ + 13200, + 16400, + 24400, + 32000, + 48000, + 64000, + 80000, + 13200, + 16400, + 24400, + 32000, + 48000, + 64000, + ], dtx=7 * [False] + 6 * [True], - testsets=["a", "d"] + testsets=["a", "d"], ), - ExperimentParams( - name="BS1534-1a", - n_conditions=2, - bitrates=[48000, 64000], - testsets=["a", "d"] + name="BS1534-1a", n_conditions=2, bitrates=[48000, 64000], testsets=["a", "d"] ), ExperimentParams( - name="BS1534-1b", - n_conditions=2, - bitrates=[96000, 128000], - testsets=["b", "d"] + name="BS1534-1b", n_conditions=2, bitrates=[96000, 128000], testsets=["b", "d"] ), ExperimentParams( - name="BS1534-2a", - n_conditions=2, - bitrates=[64000, 96000], - testsets=["a", "b"] + name="BS1534-2a", n_conditions=2, bitrates=[64000, 96000], testsets=["a", "b"] ), ExperimentParams( - name="BS1534-2b", - n_conditions=2, - bitrates=[128000, 160000], - testsets=["a", "b"] + name="BS1534-2b", n_conditions=2, bitrates=[128000, 160000], testsets=["a", "b"] ), ExperimentParams( - name="BS1534-3a", - n_conditions=2, - bitrates=[128000, 160000], - testsets=["a", "b"] + name="BS1534-3a", n_conditions=2, bitrates=[128000, 160000], testsets=["a", "b"] ), ExperimentParams( - name="BS1534-3b", - n_conditions=2, - bitrates=[384000, 512000], - testsets=["a", "b"] + name="BS1534-3b", n_conditions=2, bitrates=[384000, 512000], testsets=["a", "b"] ), ExperimentParams( name="BS1534-4a", n_conditions=3, bitrates=[96000, 128000, 160000], - testsets=["a", "d"] + testsets=["a", "d"], ), ExperimentParams( - name="BS1534-4b", - n_conditions=2, - bitrates=[160000, 192000], - testsets=["b", "d"] + name="BS1534-4b", n_conditions=2, bitrates=[160000, 192000], testsets=["b", "d"] ), ExperimentParams( - name="BS1534-5a", - n_conditions=2, - bitrates=[192000, 256000], - testsets=["a", "d"] + name="BS1534-5a", n_conditions=2, bitrates=[192000, 256000], testsets=["a", "d"] ), ExperimentParams( - name="BS1534-5b", - n_conditions=2, - bitrates=[384000, 512000], - testsets=["a", "b"] + name="BS1534-5b", n_conditions=2, bitrates=[384000, 512000], testsets=["a", "b"] ), ExperimentParams( name="BS1534-6a", n_conditions=3, bitrates=[48000, 64000, 96000], - testsets=["b", "d"] + testsets=["b", "d"], ), ExperimentParams( name="BS1534-6b", n_conditions=3, bitrates=[96000, 128000, 256000], - testsets=["b", "d"] + testsets=["b", "d"], ), ExperimentParams( - name="BS1534-7a", - n_conditions=2, - bitrates=[96000, 128000], - testsets=["b", "d"] + name="BS1534-7a", n_conditions=2, bitrates=[96000, 128000], testsets=["b", "d"] ), ExperimentParams( - name="BS1534-7b", - n_conditions=2, - bitrates=[192000, 256000], - testsets=["b", "d"] + name="BS1534-7b", n_conditions=2, bitrates=[192000, 256000], testsets=["b", "d"] ), ] diff --git a/tests/codec_be_on_mr_selection/test_experiments.py b/tests/codec_be_on_mr_selection/test_experiments.py index 5552d83e91..2f2683b6c2 100644 --- a/tests/codec_be_on_mr_selection/test_experiments.py +++ b/tests/codec_be_on_mr_selection/test_experiments.py @@ -3,11 +3,33 @@ from . import run_check from .constants import EXPERIMENT_PARAMS, P800_CATEGORIES -P800_PARAMS_UNIFIED = [i for sl in [p.assemble_unified_params() for p in EXPERIMENT_PARAMS] for i in sl] +TEST_PARAMS = [ + i for sl in [p.assemble_unified_params() for p in EXPERIMENT_PARAMS] for i in sl +] @pytest.mark.create_ref -@pytest.mark.parametrize("experiment,bitrate,dtx,fer,testset", P800_PARAMS_UNIFIED) +@pytest.mark.parametrize("experiment,bitrate,dtx,fer,testset", TEST_PARAMS) @pytest.mark.parametrize("category", P800_CATEGORIES) -def test_experiment(experiment, bitrate, dtx, fer, testset, category, dut_encoder_frontend, dut_decoder_frontend, update_ref): - run_check(experiment, category, testset, bitrate, dtx, fer, dut_encoder_frontend, dut_decoder_frontend, update_ref == 1) +def test_experiment( + experiment, + bitrate, + dtx, + fer, + testset, + category, + dut_encoder_frontend, + dut_decoder_frontend, + update_ref, +): + run_check( + experiment, + category, + testset, + bitrate, + dtx, + fer, + dut_encoder_frontend, + dut_decoder_frontend, + update_ref == 1, + ) -- GitLab From bb0d68a0e6bd0245b6cd7d0b1899826f830a3e31 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 6 Jul 2023 11:06:53 +0200 Subject: [PATCH 07/20] split between P800 and BS1534 tests --- tests/codec_be_on_mr_selection/__init__.py | 4 +- tests/codec_be_on_mr_selection/constants.py | 12 +++- .../test_experiments.py | 71 +++++++++++++++++-- 3 files changed, 78 insertions(+), 9 deletions(-) diff --git a/tests/codec_be_on_mr_selection/__init__.py b/tests/codec_be_on_mr_selection/__init__.py index 714de01924..64d2e1c363 100644 --- a/tests/codec_be_on_mr_selection/__init__.py +++ b/tests/codec_be_on_mr_selection/__init__.py @@ -20,7 +20,7 @@ MASA_TESTS = ("P800-8", "P800-9", "BS1534-7a", "BS1534-7a") def get_testvectors_for_exp_cat_and_testset(experiment, category, testset): - fname = f"{experiment}-cat{category}-{testset}-input.wav" + fname = f"{experiment}-{category}-{testset}-input.wav" signal = TESTV_PATH.joinpath(fname).absolute() # will be an empty list if not ISM experiment @@ -47,7 +47,7 @@ def get_out_bitstream_and_synthesis_name(input_signal, bitrate, dtx, fer): def get_error_pattern_for_exp_cat_and_testset(experiment, category, testset): - ep_file = TESTV_PATH.joinpath(f"{experiment}-cat{category}-{testset}-ep.192") + ep_file = TESTV_PATH.joinpath(f"{experiment}-{category}-{testset}-ep.192") return ep_file diff --git a/tests/codec_be_on_mr_selection/constants.py b/tests/codec_be_on_mr_selection/constants.py index f1fb94dbbb..e5ac6cc3f5 100644 --- a/tests/codec_be_on_mr_selection/constants.py +++ b/tests/codec_be_on_mr_selection/constants.py @@ -32,9 +32,11 @@ class ExperimentParams: return [(self.name, *p, t) for p, t in product(params, self.testsets)] -P800_CATEGORIES = range(1, 7) +P800_CATEGORIES = [f"cat{i}" for i in range(1, 7)] +BS1534_CATEGORIES = [""] +BS1534_MASA_CATEGORIES = ["FOA", "HOA2"] -EXPERIMENT_PARAMS = [ +P800_PARAMS = [ ExperimentParams( name="P800-1", n_conditions=12, @@ -224,6 +226,9 @@ EXPERIMENT_PARAMS = [ dtx=7 * [False] + 6 * [True], testsets=["a", "d"], ), +] + +BS1534_PARAMS = [ ExperimentParams( name="BS1534-1a", n_conditions=2, bitrates=[48000, 64000], testsets=["a", "d"] ), @@ -269,6 +274,9 @@ EXPERIMENT_PARAMS = [ bitrates=[96000, 128000, 256000], testsets=["b", "d"], ), +] + +BS1534_PARAMS_MASA = [ ExperimentParams( name="BS1534-7a", n_conditions=2, bitrates=[96000, 128000], testsets=["b", "d"] ), diff --git a/tests/codec_be_on_mr_selection/test_experiments.py b/tests/codec_be_on_mr_selection/test_experiments.py index 2f2683b6c2..885874b046 100644 --- a/tests/codec_be_on_mr_selection/test_experiments.py +++ b/tests/codec_be_on_mr_selection/test_experiments.py @@ -1,17 +1,78 @@ import pytest from . import run_check -from .constants import EXPERIMENT_PARAMS, P800_CATEGORIES +from .constants import P800_CATEGORIES, BS1534_CATEGORIES, BS1534_MASA_CATEGORIES, P800_PARAMS, BS1534_PARAMS, BS1534_PARAMS_MASA -TEST_PARAMS = [ - i for sl in [p.assemble_unified_params() for p in EXPERIMENT_PARAMS] for i in sl +P800_PARAMS_UNIFIED = [ + i for sl in [p.assemble_unified_params() for p in P800_PARAMS] for i in sl +] +BS1534_PARAMS_UNIFIED = [ + i for sl in [p.assemble_unified_params() for p in BS1534_PARAMS] for i in sl +] +BS1534_MASA_PARAMS_UNIFIED = [ + i for sl in [p.assemble_unified_params() for p in BS1534_PARAMS_MASA] for i in sl ] @pytest.mark.create_ref -@pytest.mark.parametrize("experiment,bitrate,dtx,fer,testset", TEST_PARAMS) +@pytest.mark.parametrize("experiment,bitrate,dtx,fer,testset", P800_PARAMS_UNIFIED) @pytest.mark.parametrize("category", P800_CATEGORIES) -def test_experiment( +def test_p800( + experiment, + bitrate, + dtx, + fer, + testset, + category, + dut_encoder_frontend, + dut_decoder_frontend, + update_ref, +): + run_check( + experiment, + category, + testset, + bitrate, + dtx, + fer, + dut_encoder_frontend, + dut_decoder_frontend, + update_ref == 1, + ) + +# need to split BS1534 tests as MASA tests have categories (FOA, HOA2) +# otherwise, it is the exact same test + +@pytest.mark.create_ref +@pytest.mark.parametrize("experiment,bitrate,dtx,fer,testset", BS1534_PARAMS_UNIFIED) +@pytest.mark.parametrize("category", BS1534_CATEGORIES) +def test_bs1534_no_masa( + experiment, + bitrate, + dtx, + fer, + testset, + category, + dut_encoder_frontend, + dut_decoder_frontend, + update_ref, +): + run_check( + experiment, + category, + testset, + bitrate, + dtx, + fer, + dut_encoder_frontend, + dut_decoder_frontend, + update_ref == 1, + ) + +@pytest.mark.create_ref +@pytest.mark.parametrize("experiment,bitrate,dtx,fer,testset", BS1534_MASA_PARAMS_UNIFIED) +@pytest.mark.parametrize("category", BS1534_MASA_CATEGORIES) +def test_bs1534_masa( experiment, bitrate, dtx, -- GitLab From 14d5eafdec3f11950866eee7db1346defd694d3e Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 6 Jul 2023 13:57:55 +0200 Subject: [PATCH 08/20] fix testvector collection --- tests/codec_be_on_mr_selection/__init__.py | 22 +++++++-- tests/codec_be_on_mr_selection/constants.py | 45 ++++++++++++++++++- .../test_experiments.py | 35 ++++++++------- 3 files changed, 81 insertions(+), 21 deletions(-) diff --git a/tests/codec_be_on_mr_selection/__init__.py b/tests/codec_be_on_mr_selection/__init__.py index 64d2e1c363..a5fc328cb7 100644 --- a/tests/codec_be_on_mr_selection/__init__.py +++ b/tests/codec_be_on_mr_selection/__init__.py @@ -19,14 +19,18 @@ ISM_NUM_FOR_EXP = { MASA_TESTS = ("P800-8", "P800-9", "BS1534-7a", "BS1534-7a") -def get_testvectors_for_exp_cat_and_testset(experiment, category, testset): - fname = f"{experiment}-{category}-{testset}-input.wav" +def get_testvectors_for_exp_cat_and_testset( + experiment, category, testset, input_file_num +): + input_file_num_str = "" if input_file_num is None else f"-{input_file_num}" + category = "-" if category == "" else f"-{category}-" + fname = f"{experiment}{category}{testset}-input{input_file_num_str}.wav" signal = TESTV_PATH.joinpath(fname).absolute() # will be an empty list if not ISM experiment metadata = sorted( [ - TESTV_PATH.joinpath(fname + f"{i}.csv") + TESTV_PATH.joinpath(fname + f".{i}.csv") for i in range(ISM_NUM_FOR_EXP.get(experiment, 0)) ] ) @@ -82,18 +86,28 @@ def run_check( encoder_frontend, decoder_frontend, is_ref_creation, + input_file_num=None, ): sampling_rate = 48 output_mode, options = OUTPUT_MODES_AND_OPTIONS_FOR_EXPERIMENT[experiment] testv, metadata = get_testvectors_for_exp_cat_and_testset( - experiment, category, testset + experiment, category, testset, input_file_num ) error_pattern = ( get_error_pattern_for_exp_cat_and_testset(experiment, category, testset) if fer else None ) + + if not testv.exists(): + pytest.fail(f"Testv file not found: {testv}") + if error_pattern is not None and not error_pattern.exists(): + pytest.fail(f"Error pattern not found: {error_pattern}") + for md in metadata: + if not md.exists(): + pytest.fail(f"MD file not found: {md}") + dut_bitstream, dut_output = get_out_bitstream_and_synthesis_name( testv, bitrate, dtx, fer ) diff --git a/tests/codec_be_on_mr_selection/constants.py b/tests/codec_be_on_mr_selection/constants.py index e5ac6cc3f5..8da5b2ec74 100644 --- a/tests/codec_be_on_mr_selection/constants.py +++ b/tests/codec_be_on_mr_selection/constants.py @@ -34,7 +34,9 @@ class ExperimentParams: P800_CATEGORIES = [f"cat{i}" for i in range(1, 7)] BS1534_CATEGORIES = [""] -BS1534_MASA_CATEGORIES = ["FOA", "HOA2"] +BS1534_MASA_HOA2_FILE_NUMS = [1, 3, 5, 7, 9, 11, 13, 15] +BS1534_MASA_FOA_FILE_NUMS = [2, 4, 6, 8, 10, 12, 14, 16] +BS1534_N_FILES = 16 P800_PARAMS = [ ExperimentParams( @@ -285,6 +287,33 @@ BS1534_PARAMS_MASA = [ ), ] +P800_PARAMS_UNIFIED = [ + i for sl in [p.assemble_unified_params() for p in P800_PARAMS] for i in sl +] +BS1534_PARAMS_UNIFIED = [ + i for sl in [p.assemble_unified_params() for p in BS1534_PARAMS] for i in sl +] + +BS1534_MASA_PARAMS = [ + i for sl in [p.assemble_unified_params() for p in BS1534_PARAMS_MASA] for i in sl +] +BS1534_MASA_PARAMS = [ + x + tuple([y]) for x, y in product(BS1534_MASA_PARAMS, ["FOA", "HOA2"]) +] +BS1534_MASA_PARAMS_UNIFIED = [ + x + tuple([y]) + for x, y in product( + [p for p in BS1534_MASA_PARAMS if p[5] == "FOA"], + range(2, 1 + BS1534_N_FILES, 2), + ) +] + [ + x + tuple([y]) + for x, y in product( + [p for p in BS1534_MASA_PARAMS if p[5] == "HOA2"], + range(1, 1 + BS1534_N_FILES, 2), + ) +] + OUTPUT_MODES_AND_OPTIONS_FOR_EXPERIMENT = { "P800-1": ("STEREO", ["-stereo"]), "P800-2": ("STEREO", ["-stereo"]), @@ -295,4 +324,18 @@ OUTPUT_MODES_AND_OPTIONS_FOR_EXPERIMENT = { "P800-7": ("EXT", ["-ism", "2"]), "P800-8": ("EXT", ["-masa", "2"]), "P800-9": ("EXT", ["-masa", "2"]), + "BS1534-1a": ("STEREO", ["-stereo"]), + "BS1534-1b": ("STEREO", ["-stereo"]), + "BS1534-2a": ("5_1", ["-mc", "5_1"]), + "BS1534-2b": ("5_1", ["-mc", "5_1"]), + "BS1534-3a": ("7_1_4", ["-mc", "7_1_4"]), + "BS1534-3b": ("7_1_4", ["-mc", "7_1_4"]), + "BS1534-4a": ("HOA3", ["-sba", "1"]), + "BS1534-4b": ("HOA3", ["-sba", "2"]), + "BS1534-5a": ("HOA3", ["-sba", "3"]), + "BS1534-5b": ("HOA3", ["-sba", "3"]), + "BS1534-6a": ("EXT", ["-ism", "3"]), + "BS1534-6b": ("EXT", ["-ism", "4"]), + "BS1534-7a": ("EXT", ["-masa", "2"]), + "BS1534-7b": ("EXT", ["-masa", "2"]), } diff --git a/tests/codec_be_on_mr_selection/test_experiments.py b/tests/codec_be_on_mr_selection/test_experiments.py index 885874b046..67405e444c 100644 --- a/tests/codec_be_on_mr_selection/test_experiments.py +++ b/tests/codec_be_on_mr_selection/test_experiments.py @@ -1,17 +1,14 @@ import pytest +from itertools import product from . import run_check -from .constants import P800_CATEGORIES, BS1534_CATEGORIES, BS1534_MASA_CATEGORIES, P800_PARAMS, BS1534_PARAMS, BS1534_PARAMS_MASA - - -P800_PARAMS_UNIFIED = [ - i for sl in [p.assemble_unified_params() for p in P800_PARAMS] for i in sl -] -BS1534_PARAMS_UNIFIED = [ - i for sl in [p.assemble_unified_params() for p in BS1534_PARAMS] for i in sl -] -BS1534_MASA_PARAMS_UNIFIED = [ - i for sl in [p.assemble_unified_params() for p in BS1534_PARAMS_MASA] for i in sl -] +from .constants import ( + P800_CATEGORIES, + BS1534_CATEGORIES, + BS1534_N_FILES, + P800_PARAMS_UNIFIED, + BS1534_PARAMS_UNIFIED, + BS1534_MASA_PARAMS_UNIFIED, +) @pytest.mark.create_ref @@ -40,12 +37,11 @@ def test_p800( update_ref == 1, ) -# need to split BS1534 tests as MASA tests have categories (FOA, HOA2) -# otherwise, it is the exact same test @pytest.mark.create_ref @pytest.mark.parametrize("experiment,bitrate,dtx,fer,testset", BS1534_PARAMS_UNIFIED) @pytest.mark.parametrize("category", BS1534_CATEGORIES) +@pytest.mark.parametrize("input_file_num", range(1, 1 + BS1534_N_FILES)) def test_bs1534_no_masa( experiment, bitrate, @@ -53,6 +49,7 @@ def test_bs1534_no_masa( fer, testset, category, + input_file_num, dut_encoder_frontend, dut_decoder_frontend, update_ref, @@ -67,11 +64,15 @@ def test_bs1534_no_masa( dut_encoder_frontend, dut_decoder_frontend, update_ref == 1, + input_file_num=input_file_num, ) + @pytest.mark.create_ref -@pytest.mark.parametrize("experiment,bitrate,dtx,fer,testset", BS1534_MASA_PARAMS_UNIFIED) -@pytest.mark.parametrize("category", BS1534_MASA_CATEGORIES) +@pytest.mark.parametrize( + "experiment,bitrate,dtx,fer,testset,category,input_file_num", + BS1534_MASA_PARAMS_UNIFIED, +) def test_bs1534_masa( experiment, bitrate, @@ -79,6 +80,7 @@ def test_bs1534_masa( fer, testset, category, + input_file_num, dut_encoder_frontend, dut_decoder_frontend, update_ref, @@ -93,4 +95,5 @@ def test_bs1534_masa( dut_encoder_frontend, dut_decoder_frontend, update_ref == 1, + input_file_num=input_file_num, ) -- GitLab From fef6277b9041376525ca2b7e4b68ca5314398228 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 6 Jul 2023 14:55:26 +0200 Subject: [PATCH 09/20] dont't use paths for MD files, but str in option list --- tests/codec_be_on_mr_selection/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/codec_be_on_mr_selection/__init__.py b/tests/codec_be_on_mr_selection/__init__.py index a5fc328cb7..09f1d75cf7 100644 --- a/tests/codec_be_on_mr_selection/__init__.py +++ b/tests/codec_be_on_mr_selection/__init__.py @@ -112,7 +112,7 @@ def run_check( testv, bitrate, dtx, fer ) - options.extend(metadata) + options.extend([str(f) for f in metadata]) encoder_frontend.run( bitrate, sampling_rate, -- GitLab From 592d85fd226369f2d42b301a396049e7701e76f9 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 6 Jul 2023 15:39:58 +0200 Subject: [PATCH 10/20] fix bug with mutating a constant --- tests/codec_be_on_mr_selection/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/codec_be_on_mr_selection/__init__.py b/tests/codec_be_on_mr_selection/__init__.py index 09f1d75cf7..6dad2ee3f5 100644 --- a/tests/codec_be_on_mr_selection/__init__.py +++ b/tests/codec_be_on_mr_selection/__init__.py @@ -37,6 +37,8 @@ def get_testvectors_for_exp_cat_and_testset( if experiment in MASA_TESTS: metadata.append(TESTV_PATH.joinpath(fname + ".met")) + print(metadata) + return signal, metadata @@ -112,14 +114,13 @@ def run_check( testv, bitrate, dtx, fer ) - options.extend([str(f) for f in metadata]) encoder_frontend.run( bitrate, sampling_rate, testv, dut_bitstream, dtx_mode=dtx, - add_option_list=options, + add_option_list=options + [str(f) for f in metadata], ) if not is_ref_creation and not is_be_to_reference(dut_bitstream): -- GitLab From 872a9f92b1560222b11dfbb76f7227ade877e009 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 7 Jul 2023 08:53:14 +0200 Subject: [PATCH 11/20] fix typo --- tests/codec_be_on_mr_selection/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/codec_be_on_mr_selection/__init__.py b/tests/codec_be_on_mr_selection/__init__.py index 6dad2ee3f5..d6e368068d 100644 --- a/tests/codec_be_on_mr_selection/__init__.py +++ b/tests/codec_be_on_mr_selection/__init__.py @@ -16,7 +16,7 @@ ISM_NUM_FOR_EXP = { "BS1534-6a": 3, "BS1534-6b": 4, } -MASA_TESTS = ("P800-8", "P800-9", "BS1534-7a", "BS1534-7a") +MASA_TESTS = ("P800-8", "P800-9", "BS1534-7a", "BS1534-7b") def get_testvectors_for_exp_cat_and_testset( -- GitLab From 9b749e19b10917d839f0f6a44f1f80838608d246 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 7 Jul 2023 09:26:08 +0200 Subject: [PATCH 12/20] don't overwrite bitstream when applying frame errors --- tests/codec_be_on_mr_selection/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/codec_be_on_mr_selection/__init__.py b/tests/codec_be_on_mr_selection/__init__.py index d6e368068d..9eeaa590be 100644 --- a/tests/codec_be_on_mr_selection/__init__.py +++ b/tests/codec_be_on_mr_selection/__init__.py @@ -126,10 +126,12 @@ def run_check( if not is_ref_creation and not is_be_to_reference(dut_bitstream): pytest.fail("Bitstream file differs from reference") + dut_bitstream_to_decoder = dut_bitstream if error_pattern is not None: - apply_error_pattern_on_bitstream(dut_bitstream, error_pattern, dut_bitstream) + dut_bitstream_to_decoder = dut_bitstream.with_suffix(".err" + dut_bitstream.suffix) + apply_error_pattern_on_bitstream(dut_bitstream, error_pattern, dut_bitstream_to_decoder) - decoder_frontend.run(output_mode, sampling_rate, dut_bitstream, dut_output) + decoder_frontend.run(output_mode, sampling_rate, dut_bitstream_to_decoder, dut_output) # TODO: also compare metadata if present if not is_ref_creation and not is_be_to_reference(dut_output): -- GitLab From e0ce3145ccf78b379310c9fec2381ad41511bea3 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 7 Jul 2023 10:39:07 +0200 Subject: [PATCH 13/20] also compare metadata files for BE --- tests/codec_be_on_mr_selection/__init__.py | 27 ++++++++++++++-------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/tests/codec_be_on_mr_selection/__init__.py b/tests/codec_be_on_mr_selection/__init__.py index 9eeaa590be..1bba1961a2 100644 --- a/tests/codec_be_on_mr_selection/__init__.py +++ b/tests/codec_be_on_mr_selection/__init__.py @@ -70,12 +70,8 @@ def apply_error_pattern_on_bitstream( subprocess.run(cmd) -def is_be_to_reference(dut_file: Path): - assert dut_file.parent == DUT_PATH - - # TODO: handle .wav files differently - ref_file = REF_PATH.joinpath(dut_file.name) - return filecmp.cmp(ref_file, dut_file) +def files_equal(dut_file, ref_file): + return filecmp.cmp(dut_file, ref_file) def run_check( @@ -123,7 +119,8 @@ def run_check( add_option_list=options + [str(f) for f in metadata], ) - if not is_ref_creation and not is_be_to_reference(dut_bitstream): + ref_bitstream = REF_PATH.joinpath(dut_bitstream.name) + if not is_ref_creation and not files_equal(dut_bitstream, ref_bitstream): pytest.fail("Bitstream file differs from reference") dut_bitstream_to_decoder = dut_bitstream @@ -133,6 +130,16 @@ def run_check( decoder_frontend.run(output_mode, sampling_rate, dut_bitstream_to_decoder, dut_output) - # TODO: also compare metadata if present - if not is_ref_creation and not is_be_to_reference(dut_output): - pytest.fail("Decoder output differs from reference") + # NOTE: file comparison is done via filecmp which compares the whole file, including the header for wav files + # this should not be a problem as both the reference and the tdut output was generated by the codec, so + # diverging headers should also indicate a problem - still, keep in mind if something bogus happens + if not is_ref_creation: + ref_output = REF_PATH.joinpath(dut_output.name) + if not files_equal(dut_output, ref_output): + pytest.fail("Decoder output differs from reference") + for md in metadata: + md_suffix = "".join(md.suffixes) + dut_md = DUT_PATH.joinpath(dut_output.with_suffix(md_suffix).name) + ref_md = REF_PATH.joinpath(dut_output.with_suffix(md_suffix).name) + if not files_equal(dut_md, ref_md): + pytest.fail("Metadata file {md.name} differs from reference") -- GitLab From c0d849df2c0f2f1e4700f1f8d650d19c0eee2250 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 7 Jul 2023 11:41:28 +0200 Subject: [PATCH 14/20] add copyright headers --- tests/codec_be_on_mr_selection/__init__.py | 31 ++++++++++++++++++ tests/codec_be_on_mr_selection/constants.py | 31 ++++++++++++++++++ .../test_experiments.py | 32 ++++++++++++++++++- 3 files changed, 93 insertions(+), 1 deletion(-) diff --git a/tests/codec_be_on_mr_selection/__init__.py b/tests/codec_be_on_mr_selection/__init__.py index 1bba1961a2..cb56a0caff 100644 --- a/tests/codec_be_on_mr_selection/__init__.py +++ b/tests/codec_be_on_mr_selection/__init__.py @@ -1,3 +1,34 @@ +__copyright__ = \ +""" +(C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + import tempfile import pytest import filecmp diff --git a/tests/codec_be_on_mr_selection/constants.py b/tests/codec_be_on_mr_selection/constants.py index 8da5b2ec74..58a4911411 100644 --- a/tests/codec_be_on_mr_selection/constants.py +++ b/tests/codec_be_on_mr_selection/constants.py @@ -1,3 +1,34 @@ +__copyright__ = \ +""" +(C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + from itertools import product from typing import Union, List diff --git a/tests/codec_be_on_mr_selection/test_experiments.py b/tests/codec_be_on_mr_selection/test_experiments.py index 67405e444c..b02fd4f0b8 100644 --- a/tests/codec_be_on_mr_selection/test_experiments.py +++ b/tests/codec_be_on_mr_selection/test_experiments.py @@ -1,5 +1,35 @@ +__copyright__ = \ +""" +(C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + import pytest -from itertools import product from . import run_check from .constants import ( P800_CATEGORIES, -- GitLab From 3a334e5699bc721aff2be662354a9f93fb2f7e1e Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 7 Jul 2023 11:49:26 +0200 Subject: [PATCH 15/20] cleanup, formatting, comments --- tests/codec_be_on_mr_selection/__init__.py | 20 +++++++++------- tests/codec_be_on_mr_selection/constants.py | 23 +++++++++++++------ .../test_experiments.py | 3 +-- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/tests/codec_be_on_mr_selection/__init__.py b/tests/codec_be_on_mr_selection/__init__.py index cb56a0caff..8e5951648f 100644 --- a/tests/codec_be_on_mr_selection/__init__.py +++ b/tests/codec_be_on_mr_selection/__init__.py @@ -1,5 +1,4 @@ -__copyright__ = \ -""" +__copyright__ = """ (C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, @@ -97,7 +96,6 @@ def apply_error_pattern_on_bitstream( in_bitstream = Path(tmpdir).joinpath(in_bitstream.name) cmd = ["eid-xor", "-vbr", "-fer", in_bitstream, error_pattern, out_bitstream] - subprocess.run(cmd) @@ -156,13 +154,19 @@ def run_check( dut_bitstream_to_decoder = dut_bitstream if error_pattern is not None: - dut_bitstream_to_decoder = dut_bitstream.with_suffix(".err" + dut_bitstream.suffix) - apply_error_pattern_on_bitstream(dut_bitstream, error_pattern, dut_bitstream_to_decoder) - - decoder_frontend.run(output_mode, sampling_rate, dut_bitstream_to_decoder, dut_output) + dut_bitstream_to_decoder = dut_bitstream.with_suffix( + ".err" + dut_bitstream.suffix + ) + apply_error_pattern_on_bitstream( + dut_bitstream, error_pattern, dut_bitstream_to_decoder + ) + + decoder_frontend.run( + output_mode, sampling_rate, dut_bitstream_to_decoder, dut_output + ) # NOTE: file comparison is done via filecmp which compares the whole file, including the header for wav files - # this should not be a problem as both the reference and the tdut output was generated by the codec, so + # this should not be a problem as both the reference and the tdut output was generated by the codec, so # diverging headers should also indicate a problem - still, keep in mind if something bogus happens if not is_ref_creation: ref_output = REF_PATH.joinpath(dut_output.name) diff --git a/tests/codec_be_on_mr_selection/constants.py b/tests/codec_be_on_mr_selection/constants.py index 58a4911411..3536cb6cb6 100644 --- a/tests/codec_be_on_mr_selection/constants.py +++ b/tests/codec_be_on_mr_selection/constants.py @@ -1,5 +1,4 @@ -__copyright__ = \ -""" +__copyright__ = """ (C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, @@ -34,6 +33,11 @@ from typing import Union, List class ExperimentParams: + """ + Class to bundle the parameters for the experiments. Main purpose is to make things more readable over + just using a lot of lists and bundling sanitiy checks and needed utilities together. + """ + def __init__( self, name: str, @@ -59,6 +63,10 @@ class ExperimentParams: self.fer = fer def assemble_unified_params(self): + """ + Create the parameter combinations for the tested operating points. + Just passing them to pytest would give too many (and not tested) combinations, so do it here. + """ params = zip(self.bitrates, self.dtx, self.fer) return [(self.name, *p, t) for p, t in product(params, self.testsets)] @@ -318,6 +326,8 @@ BS1534_PARAMS_MASA = [ ), ] +# These "_UNIFIED" parameter lists are the thigns that get passed to the pytest parametrization + P800_PARAMS_UNIFIED = [ i for sl in [p.assemble_unified_params() for p in P800_PARAMS] for i in sl ] @@ -325,20 +335,19 @@ BS1534_PARAMS_UNIFIED = [ i for sl in [p.assemble_unified_params() for p in BS1534_PARAMS] for i in sl ] +# for the MASA BS1534 tests everything is complicated because of categories and different numbering in them, hence the extra code here BS1534_MASA_PARAMS = [ i for sl in [p.assemble_unified_params() for p in BS1534_PARAMS_MASA] for i in sl ] -BS1534_MASA_PARAMS = [ - x + tuple([y]) for x, y in product(BS1534_MASA_PARAMS, ["FOA", "HOA2"]) -] +BS1534_MASA_PARAMS = [x + (y,) for x, y in product(BS1534_MASA_PARAMS, ["FOA", "HOA2"])] BS1534_MASA_PARAMS_UNIFIED = [ - x + tuple([y]) + x + (y,) for x, y in product( [p for p in BS1534_MASA_PARAMS if p[5] == "FOA"], range(2, 1 + BS1534_N_FILES, 2), ) ] + [ - x + tuple([y]) + x + (y,) for x, y in product( [p for p in BS1534_MASA_PARAMS if p[5] == "HOA2"], range(1, 1 + BS1534_N_FILES, 2), diff --git a/tests/codec_be_on_mr_selection/test_experiments.py b/tests/codec_be_on_mr_selection/test_experiments.py index b02fd4f0b8..cbc3f9d4cd 100644 --- a/tests/codec_be_on_mr_selection/test_experiments.py +++ b/tests/codec_be_on_mr_selection/test_experiments.py @@ -1,5 +1,4 @@ -__copyright__ = \ -""" +__copyright__ = """ (C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, -- GitLab From 9a960a147fbc8709995fa8f704c7a039f6a9cf9d Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 7 Jul 2023 11:57:23 +0200 Subject: [PATCH 16/20] add empty folders so they are there for CI --- tests/codec_be_on_mr_selection/dut/.gitkeep | 0 tests/codec_be_on_mr_selection/ref/.gitkeep | 0 tests/codec_be_on_mr_selection/testv/.gitkeep | 0 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/codec_be_on_mr_selection/dut/.gitkeep create mode 100644 tests/codec_be_on_mr_selection/ref/.gitkeep create mode 100644 tests/codec_be_on_mr_selection/testv/.gitkeep diff --git a/tests/codec_be_on_mr_selection/dut/.gitkeep b/tests/codec_be_on_mr_selection/dut/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/codec_be_on_mr_selection/ref/.gitkeep b/tests/codec_be_on_mr_selection/ref/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/codec_be_on_mr_selection/testv/.gitkeep b/tests/codec_be_on_mr_selection/testv/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 -- GitLab From b83717700fadda370ff92df162ff80421a2ebdfa Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 7 Jul 2023 12:05:24 +0200 Subject: [PATCH 17/20] fix tests --- .gitlab-ci.yml | 2 +- tests/conftest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fe829be035..ef20a67e5f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1128,7 +1128,7 @@ coverage-test-on-main-scheduled: - python3 tests/create_short_testvectors.py - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n 0 --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n 0 --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - - python3 -m pytest tests/test_param_file.py -v -n 0 --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n 0 --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - bash ci/smoke_test.sh coverage - python3 -m pytest -q -n auto tests/renderer/test_renderer_be_comparison.py - bash ci/ivas_voip_be_test.sh coverage diff --git a/tests/conftest.py b/tests/conftest.py index 958e0b10ac..12b62caccd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,7 +41,7 @@ from subprocess import run import textwrap from typing import Optional import os -import tests.testconfig +from tests import testconfig import pytest logger = logging.getLogger(__name__) -- GitLab From 6cde790eb888c20b0453a5ae714c0b532f08abdd Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 7 Jul 2023 15:40:38 +0200 Subject: [PATCH 18/20] use environment variables for setting the ref and testv folders --- tests/codec_be_on_mr_selection/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/codec_be_on_mr_selection/__init__.py b/tests/codec_be_on_mr_selection/__init__.py index 8e5951648f..9dae2e2281 100644 --- a/tests/codec_be_on_mr_selection/__init__.py +++ b/tests/codec_be_on_mr_selection/__init__.py @@ -30,14 +30,17 @@ the United Nations Convention on Contracts on the International Sales of Goods. import tempfile import pytest +import os import filecmp from pathlib import Path import subprocess from .constants import OUTPUT_MODES_AND_OPTIONS_FOR_EXPERIMENT HERE = Path(__file__).parent -TESTV_PATH = HERE.joinpath("testv") -REF_PATH = HERE.joinpath("ref") +# set environment variables in CI job +# need to wrap in path again as env var is string +TESTV_PATH = Path(os.environ.get("SELECTION_BE_TESTV_DIR", HERE.joinpath("testv"))) +REF_PATH = Path(os.environ.get("SELECTION_BE_REF_DIR", HERE.joinpath("ref"))) DUT_PATH = HERE.joinpath("dut") ISM_NUM_FOR_EXP = { -- GitLab From a9be5aa0edb15b6c5e0b8a30db7e5210dbc8b930 Mon Sep 17 00:00:00 2001 From: kiene Date: Fri, 7 Jul 2023 16:22:02 +0200 Subject: [PATCH 19/20] no need to parametrize only one category for BS1534 tests (except for MASA tests) --- tests/codec_be_on_mr_selection/constants.py | 1 - tests/codec_be_on_mr_selection/test_experiments.py | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/codec_be_on_mr_selection/constants.py b/tests/codec_be_on_mr_selection/constants.py index 3536cb6cb6..12d8d68dab 100644 --- a/tests/codec_be_on_mr_selection/constants.py +++ b/tests/codec_be_on_mr_selection/constants.py @@ -72,7 +72,6 @@ class ExperimentParams: P800_CATEGORIES = [f"cat{i}" for i in range(1, 7)] -BS1534_CATEGORIES = [""] BS1534_MASA_HOA2_FILE_NUMS = [1, 3, 5, 7, 9, 11, 13, 15] BS1534_MASA_FOA_FILE_NUMS = [2, 4, 6, 8, 10, 12, 14, 16] BS1534_N_FILES = 16 diff --git a/tests/codec_be_on_mr_selection/test_experiments.py b/tests/codec_be_on_mr_selection/test_experiments.py index cbc3f9d4cd..a0a4aa780a 100644 --- a/tests/codec_be_on_mr_selection/test_experiments.py +++ b/tests/codec_be_on_mr_selection/test_experiments.py @@ -32,7 +32,6 @@ import pytest from . import run_check from .constants import ( P800_CATEGORIES, - BS1534_CATEGORIES, BS1534_N_FILES, P800_PARAMS_UNIFIED, BS1534_PARAMS_UNIFIED, @@ -69,7 +68,6 @@ def test_p800( @pytest.mark.create_ref @pytest.mark.parametrize("experiment,bitrate,dtx,fer,testset", BS1534_PARAMS_UNIFIED) -@pytest.mark.parametrize("category", BS1534_CATEGORIES) @pytest.mark.parametrize("input_file_num", range(1, 1 + BS1534_N_FILES)) def test_bs1534_no_masa( experiment, @@ -77,12 +75,12 @@ def test_bs1534_no_masa( dtx, fer, testset, - category, input_file_num, dut_encoder_frontend, dut_decoder_frontend, update_ref, ): + category = "" run_check( experiment, category, -- GitLab From 9a6d2967e820105bb7b8e19e7b9d999b352f2dc1 Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 10 Jul 2023 11:54:48 +0200 Subject: [PATCH 20/20] cleanup and make parametrization more readable --- tests/codec_be_on_mr_selection/__init__.py | 12 +++---- tests/codec_be_on_mr_selection/constants.py | 40 ++++++++++++--------- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/tests/codec_be_on_mr_selection/__init__.py b/tests/codec_be_on_mr_selection/__init__.py index 9dae2e2281..2302cec387 100644 --- a/tests/codec_be_on_mr_selection/__init__.py +++ b/tests/codec_be_on_mr_selection/__init__.py @@ -34,7 +34,7 @@ import os import filecmp from pathlib import Path import subprocess -from .constants import OUTPUT_MODES_AND_OPTIONS_FOR_EXPERIMENT +from .constants import OUTPUT_MODES_AND_OPTIONS_FOR_EXPERIMENT, DTX_ON, FER_5PERC HERE = Path(__file__).parent # set environment variables in CI job @@ -70,17 +70,13 @@ def get_testvectors_for_exp_cat_and_testset( if experiment in MASA_TESTS: metadata.append(TESTV_PATH.joinpath(fname + ".met")) - print(metadata) - return signal, metadata def get_out_bitstream_and_synthesis_name(input_signal, bitrate, dtx, fer): - dtx_str = "DTXon" if dtx else "DTXoff" - fer_str = "FER_5perc" if fer else "FER_0perc" suffix = [".192", ".wav"] return [ - DUT_PATH.joinpath(f"{input_signal.stem}-{bitrate}bps-{dtx_str}-{fer_str}{s}") + DUT_PATH.joinpath(f"{input_signal.stem}-{bitrate}bps-{dtx}-{fer}{s}") for s in suffix ] @@ -126,7 +122,7 @@ def run_check( ) error_pattern = ( get_error_pattern_for_exp_cat_and_testset(experiment, category, testset) - if fer + if fer == FER_5PERC else None ) @@ -147,7 +143,7 @@ def run_check( sampling_rate, testv, dut_bitstream, - dtx_mode=dtx, + dtx_mode=True if dtx == DTX_ON else False, add_option_list=options + [str(f) for f in metadata], ) diff --git a/tests/codec_be_on_mr_selection/constants.py b/tests/codec_be_on_mr_selection/constants.py index 12d8d68dab..e0afebc5d6 100644 --- a/tests/codec_be_on_mr_selection/constants.py +++ b/tests/codec_be_on_mr_selection/constants.py @@ -32,6 +32,12 @@ from itertools import product from typing import Union, List +DTX_ON = "DTXon" +DTX_OFF = "DTXoff" +FER_5PERC = "FER_5perc" +FER_0PERC = "FER_0perc" + + class ExperimentParams: """ Class to bundle the parameters for the experiments. Main purpose is to make things more readable over @@ -44,11 +50,11 @@ class ExperimentParams: n_conditions: int, bitrates: List[int], testsets: List[str], - dtx: Union[List[bool], bool] = False, - fer: Union[List[bool], bool] = False, + dtx: Union[List[str], str] = DTX_OFF, + fer: Union[List[str], str] = FER_0PERC, ): - dtx = n_conditions * [dtx] if isinstance(dtx, bool) else dtx - fer = n_conditions * [fer] if isinstance(fer, bool) else fer + dtx = n_conditions * [dtx] if isinstance(dtx, str) else dtx + fer = n_conditions * [fer] if isinstance(fer, str) else fer assert ( len(bitrates) == n_conditions and len(testsets) == 2 @@ -94,8 +100,8 @@ P800_PARAMS = [ 24400, 13200, ], - dtx=10 * [False] + 2 * [True], - fer=5 * [False] + 5 * [True] + [False, True], + dtx=10 * [DTX_OFF] + 2 * [DTX_ON], + fer=5 * [FER_0PERC] + 5 * [FER_5PERC] + [FER_0PERC, FER_5PERC], testsets=["a", "d"], ), ExperimentParams( @@ -114,7 +120,7 @@ P800_PARAMS = [ 32000, 48000, ], - dtx=6 * [False] + 5 * [True], + dtx=6 * [DTX_OFF] + 5 * [DTX_ON], testsets=["b", "d"], ), ExperimentParams( @@ -135,8 +141,8 @@ P800_PARAMS = [ 24400, 13200, ], - dtx=11 * [False] + 2 * [True], - fer=6 * [False] + 5 * [True] + [False, True], + dtx=11 * [DTX_OFF] + 2 * [DTX_ON], + fer=6 * [FER_0PERC] + 5 * [FER_5PERC] + [FER_0PERC, FER_5PERC], testsets=["a", "d"], ), ExperimentParams( @@ -157,7 +163,7 @@ P800_PARAMS = [ 80000, 96000, ], - fer=7 * [False] + 6 * [True], + fer=7 * [FER_0PERC] + 6 * [FER_5PERC], testsets=["a", "c"], ), ExperimentParams( @@ -178,7 +184,7 @@ P800_PARAMS = [ 64000, 80000, ], - dtx=7 * [False] + 6 * [True], + dtx=7 * [DTX_OFF] + 6 * [DTX_ON], testsets=["a", "b"], ), ExperimentParams( @@ -199,8 +205,8 @@ P800_PARAMS = [ 16400, 24400, ], - dtx=10 * [False] + 3 * [True], - fer=6 * [False] + 4 * [True] + 3 * [False], + dtx=10 * [DTX_OFF] + 3 * [DTX_ON], + fer=6 * [FER_0PERC] + 4 * [FER_5PERC] + 3 * [FER_0PERC], testsets=["a", "c"], ), ExperimentParams( @@ -221,8 +227,8 @@ P800_PARAMS = [ 32000, 48000, ], - dtx=9 * [False] + 4 * [True], - fer=5 * [False] + 4 * [True] + 4 * [False], + dtx=9 * [DTX_OFF] + 4 * [DTX_ON], + fer=5 * [FER_0PERC] + 4 * [FER_5PERC] + 4 * [FER_0PERC], testsets=["a", "d"], ), ExperimentParams( @@ -242,7 +248,7 @@ P800_PARAMS = [ 48000, 64000, ], - fer=7 * [False] + 5 * [True], + fer=7 * [FER_0PERC] + 5 * [FER_5PERC], testsets=["a", "b"], ), ExperimentParams( @@ -263,7 +269,7 @@ P800_PARAMS = [ 48000, 64000, ], - dtx=7 * [False] + 6 * [True], + dtx=7 * [DTX_OFF] + 6 * [DTX_ON], testsets=["a", "d"], ), ] -- GitLab