diff --git a/.gitignore b/.gitignore index 9000be58a1955f5f3b10f8b3aaa0a48f62850d65..aecd59fd31b47a426d32933287a4ecd426a80fd4 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,8 @@ scripts/ref/ scripts/test/ scripts/out/ scripts/self_test_summary.txt +tests/dut +tests/ref # Python files that pop up when running scripts __pycache__/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cbd61ad2dc2559f9e24f43b01ff8b6a52e12ccdc..a16cdc36661c684f5a01f648e5603e49c5456bd7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,6 +6,8 @@ variables: OUT_FORMATS_CHANNEL_BASED: "stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4" OUT_FORMATS_SCENE_BASED: "FOA HOA2 HOA3" OUT_FORMATS_BINAURAL: "BINAURAL BINAURAL_ROOM" + EXIT_CODE_NON_BE: 123 + EXIT_CODE_FAIL: 1 # This sets when pipelines are created. Jobs have more specific rules to restrict them. @@ -253,7 +255,7 @@ asan-on-merge-request-linux: # compare bit exactness between target and source branch -self-test-on-merge-request: +pytest-on-merge-request: extends: - .test-job-linux - .rules-merge-request @@ -295,56 +297,53 @@ self-test-on-merge-request: - mv IVAS_dec ../IVAS_dec_ref - cd .. - ### re-checkout the commit from the source branch to have up-to-date self_test.py and scripts/testv (and actually everything) + ### re-checkout the commit from the source branch to have up-to-date test scripts and test vectors (and actually everything) - git checkout $source_branch_commit_sha - ### run selftest - - ls -altr scripts/testv - - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test | tee test_output.txt - - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test scripts/config/self_test_evs.prm | tee test_output_evs.txt - ### analyse test output - # some helper variables - "|| true" to prevent failures from grep not finding anything - evs_non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[evs[ -]*non[ -]*be\]") || true - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true - - run_errors=$(cat test_output.txt test_output_evs.txt | grep -c "test conditions had run errors") || true - - bitexact=$(cat test_output.txt | grep -c "All [0-9]* tests are bitexact") || true - - bitexact_evs=$(cat test_output_evs.txt | grep -c "All [0-9]* tests are bitexact") || true - - EXIT_CODE_NON_BE=123 - - EXIT_CODE_FAIL=1 - expected_nonbe_1=0 - expected_nonbe_2=0 - - expected_nonbe_3=0 - fail_1=0 - fail_2=0 - - fail_3=0 - # check for crashes during the test, if any happened, fail the test - - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; fail_1=1; fi + ### prepare pytest + # create short test vectors + - python3 tests/create_short_testvectors.py + # rename test binaries back + - mv IVAS_cod_test IVAS_cod + - mv IVAS_dec_test IVAS_dec + # create references + - python3 -m pytest tests -v --update_ref 1 -m create_ref + - python3 -m pytest tests -v --update_ref 1 -m create_ref_part2 + - python3 -m pytest tests/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm + + ### run pytest + - exit_code=0 + - python3 -m pytest tests -v --junit-xml=report-junit.xml || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - # check for non bitexact output and store exit code to also always run the SBA pytest - - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; fail_1=1; fi - - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; expected_nonbe_1=1; fi + - if [ $zero_errors != 1 ]; then echo "Run errors in pytest"; fail_1=1; fi - # check for non bitexact EVS output - - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag == 0 ] ; then echo "Non-bitexact EVS cases without EVS-non-BE tag encountered"; fail_2=1; fi - - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag != 0 ] ; then echo "Non-bitexact EVS cases with EVS-non-BE tag encountered"; expected_nonbe_2=1; fi + - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures without non-BE tag encountered"; fail_1=1; fi + - if [ $exit_code -eq 1 ] && [ $non_be_flag != 0 ]; then echo "pytest run had failures with non-BE tag encountered"; expected_nonbe_1=1; fi - ### run SBA pytest + ### run pytest for EVS cases - exit_code=0 - - python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$? - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - - if [ $zero_errors != 1 ]; then echo "Run errors in SBA pytest"; fail_3=1; fi - - - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures without non-BE tag encountered"; fail_3=1; fi - - if [ $exit_code -eq 1 ] && [ $non_be_flag == 1 ]; then echo "pytest run had failures with non-BE tag encountered"; expected_nonbe_3=1; fi - - # Check results from all three tests - - if [ $fail_1 -eq 1 ] || [ $fail_2 -eq 1 ] || [ $fail_3 -eq 1 ]; then exit $EXIT_CODE_FAIL; fi - - if [ $expected_nonbe_1 -eq 1 ] || [ $expected_nonbe_2 -eq 1 ] || [ $expected_nonbe_3 -eq 1 ]; then exit $EXIT_CODE_NON_BE; fi + - python3 -m pytest tests/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --junit-xml=report-junit-evs.xml || exit_code=$? + - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true + + - if [ $zero_errors != 1 ]; then echo "Run errors in pytest for EVS"; fail_2=1; fi + + - if [ $exit_code -eq 1 ] && [ $evs_non_be_flag == 0 ]; then echo "Non-bitexact EVS cases without EVS-non-BE tag encountered"; fail_2=1; fi + - if [ $exit_code -eq 1 ] && [ $evs_non_be_flag != 0 ]; then echo "Non-bitexact EVS cases with EVS-non-BE tag encountered"; expected_nonbe_2=1; fi + + # Check results from both tests + - if [ $fail_1 -eq 1 ] || [ $fail_2 -eq 1 ]; then exit $EXIT_CODE_FAIL; fi + - if [ $expected_nonbe_1 -eq 1 ] || [ $expected_nonbe_2 -eq 1 ]; then exit $EXIT_CODE_NON_BE; fi - exit 0 - + allow_failure: exit_codes: - 123 @@ -352,14 +351,13 @@ self-test-on-merge-request: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" when: always paths: - - test_output.txt - - test_output_evs.txt - - scripts/test/logs/ - - scripts/ref/logs/ - report-junit.xml - expose_as: 'Self test results' + - report-junit-evs.xml + expose_as: 'pytest results' reports: - junit: report-junit.xml + junit: + - report-junit.xml + - report-junit-evs.xml # --------------------------------------------------------------- @@ -436,37 +434,26 @@ codec-comparison-on-main-push: ### re-checkout the latest commit in the main branch - git checkout $latest_commit - ### run selftest - - ls -altr scripts/testv - - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test | tee test_output.txt - - ### analyse test output - - # some helper variables - "|| true" to prevent failures from grep not finding anything + # helper variable - "|| true" to prevent failures from grep not finding anything - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true - - run_errors=$(cat test_output.txt | grep -c "test conditions had run errors") || true - - bitexact=$(cat test_output.txt | grep -c "All [0-9]* tests are bitexact") || true - - EXIT_CODE_NON_BE=123 - - EXIT_CODE_FAIL=1 - - selftest_exit_code=0 - - # check for crashes during the test, if any happened, fail the test - - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; exit $EXIT_CODE_FAIL; fi - - # check for non bitexact output and store exit code to also always run the SBA pytest - - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_FAIL; fi - - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_NON_BE; fi - - ### run SBA pytest + ### prepare pytest + # create short test vectors + - python3 tests/create_short_testvectors.py + # rename test binaries back + - mv IVAS_cod_test IVAS_cod + - mv IVAS_dec_test IVAS_dec + # create references + - python3 -m pytest tests -v --update_ref 1 -m create_ref + - python3 -m pytest tests -v --update_ref 1 -m create_ref_part2 + + ### run pytest - exit_code=0 - - python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$? + - python3 -m pytest tests -v --junit-xml=report-junit.xml || exit_code=$? - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi; - # return exit code from selftest if everything went well with the pytest run - - exit $selftest_exit_code allow_failure: exit_codes: - 123 @@ -474,9 +461,6 @@ codec-comparison-on-main-push: name: "main-push--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" when: always paths: - - test_output.txt - - scripts/test/logs/ - - scripts/ref/logs/ - report-junit.xml expose_as: 'Results of comparison to previous merge commit' reports: @@ -645,9 +629,10 @@ coverage-test-on-main-scheduled: script: - *print-common-info - make GCOV=1 -j - - ./scripts/self_test.py --create -t 1 - - ./scripts/self_test.py --create -t 1 scripts/config/self_test_evs.prm - - ./scripts/ivas_pytests/self_test_b.py --create_only --numprocesses 1 --encref IVAS_cod --decref IVAS_dec --encdut IVAS_cod --decdut IVAS_dec + - python3 tests/create_short_testvectors.py + - python3 -m pytest tests -v -n 0 --update_ref 1 -m create_ref --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec + - python3 -m pytest tests -v -n 0 --update_ref 1 -m create_ref_part2 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec + - python3 -m pytest tests/test_param_file.py -v -n 0 --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec - lcov -c -d obj -o coverage.info - genhtml coverage.info -o coverage artifacts: diff --git a/pytest.ini b/pytest.ini index 9d9a3bbf8e4c108aa406026e9df3175c7aae17ad..c4ed77a9f63d20671c78bf04aab431f1d9bceff6 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,8 +1,13 @@ # pytest.ini # note: per convention, this file is placed in the root directory of the repository [pytest] -addopts = -ra --tb=short --basetemp=./tmp -v -junit_family=xunit1 +addopts = -ra --tb=short --basetemp=./tmp -n auto +# Write captured system-out log messages to JUnit report. +junit_logging = system-out +# Do not capture log information for passing tests to JUnit report. +junit_log_passing_tests = False +junit_duration_report = call +junit_family = xunit1 log_file_level = DEBUG log_format = %(asctime)s %(levelname)s %(message)s log_date_format = %Y-%m-%d %H:%M:%S diff --git a/scripts/ivas_pytests/self_test_b.py b/scripts/ivas_pytests/self_test_b.py deleted file mode 100755 index 941739435ee155375d6bc4fca10ba46ccecc09f3..0000000000000000000000000000000000000000 --- a/scripts/ivas_pytests/self_test_b.py +++ /dev/null @@ -1,273 +0,0 @@ -#!/usr/bin/env python3 - -""" - (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. -""" - -""" -Script to run the pytest tests. - -Step 1: Set the stage for the pytest run. - -Step 2: Run pytest. -""" - -import os -import sys -import argparse -import subprocess -import platform -from pathlib import Path - -sys.path.append('scripts/ivas_pytests/tests/') -from cut_pcm import cut_samples - -BIN_EXT = ".exe" if platform.system() == "Windows" else "" -HERE = Path(__file__).parent.resolve() -DEFAULT_ENCODER_DUT = str(HERE.joinpath(f"../../IVAS_cod{BIN_EXT}").resolve()) -DEFAULT_DECODER_DUT = str(HERE.joinpath(f"../../IVAS_dec{BIN_EXT}").resolve()) -DEFAULT_ENCODER_REF = str(HERE.joinpath(f"../../IVAS_cod_ref{BIN_EXT}").resolve()) -DEFAULT_DECODER_REF = str(HERE.joinpath(f"../../IVAS_dec_ref{BIN_EXT}").resolve()) -CREND_UNITTEST_REF = str(HERE.joinpath(f"tests/unit_tests/crend/IVAS_crend_unit_test_ref{BIN_EXT}").resolve()) -TEST_VECTOR_DIR = str(HERE.joinpath("../testv").resolve()) -REFERENCE_DIR = str(HERE.joinpath("ref").resolve()) -DUT_BASE_DIR = str(HERE.joinpath("dut").resolve()) - - -def build_enc_and_dec(src_dir): - """ - Build the encoder and decoder binaries. - """ - if platform.system() == "Windows": - olddir = os.getcwd() - os.chdir(src_dir) - os.chdir("Workspace_msvc") - command = ["MSBuild.exe", "Workspace_msvc.sln", "/t:Clean", "/p:configuration=Release", "/p:Platform=Win32"] - subprocess.run(command, check=True) - command = ["MSBuild.exe", "Workspace_msvc.sln", "/property:configuration=Release", "/p:Platform=Win32"] - subprocess.run(command, check=True) - os.chdir(olddir) - else: - command = ["make", "-C", src_dir, "clean"] - subprocess.run(command, check=True) - command = ["make", "-C", src_dir] - subprocess.run(command, check=True) - - -def build_crend_unittest(src_dir): - """ - Build the crend unit test binary. - """ - crend_dir = f"{src_dir}/scripts/ivas_pytests/tests/unit_tests/crend" - if platform.system() == "Windows": - olddir = os.getcwd() - os.chdir(crend_dir) - # command = ["MSBuild.exe", "ivas_crend_unit_test.sln", "/t:Clean", "/p:configuration=Release", "/p:Platform=Win32"] - # subprocess.run(command, check=True) - command = ["MSBuild.exe", "ivas_crend_unit_test.sln", "/property:configuration=Release", "/p:Platform=Win32"] - subprocess.run(command, check=True) - os.chdir(olddir) - else: - # command = ["make", "-C", src_dir, "clean"] - # subprocess.run(command, check=True) - command = ["make", "-C", src_dir, "IVAS_crend_unit_test"] - subprocess.run(command, check=True) - - -def build_dut_binaries(): - """ - Build the DUT binaries. - """ - print("Building the DUT binaries") - dut_src_dir = str(HERE.joinpath("../..").resolve()) - build_enc_and_dec(dut_src_dir) - build_crend_unittest(dut_src_dir) - - -def create_short_testvectors(): - """ - Create short (5sec) testvectors. - """ - print("Creating short (5sec) testvectors") - num_channels = "4" # currently only FOA - cut_from = "0.0" - cut_len = "5.0" - for fs in ['48', '32', '16']: - in_file = f"{TEST_VECTOR_DIR}/stvFOA{fs}c.pcm" - cut_gain = "1.0" - cut_file = f"{TEST_VECTOR_DIR}/stvFOA{fs}c_cut.pcm" - cut_samples(in_file, cut_file, num_channels, fs + "000", cut_from, cut_len, cut_gain) - cut_gain = "16.0" - cut_file = f"{TEST_VECTOR_DIR}/stvFOA{fs}c_cut_{cut_gain}.pcm" - cut_samples(in_file, cut_file, num_channels, fs + "000", cut_from, cut_len, cut_gain) - - -def main(argv): - # check for python >= 3.7 - if sys.version_info[0] < 3 or sys.version_info[1] < 7: - sys.exit("This script is written for Python >= 3.7. Found: " + platform.python_version()) - - parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument( - "--create_only", - action="store_true", - default=False, - help="Create references when needed, but don't run the tests" - ) - parser.add_argument( - "--numprocesses", - action="store", - default="auto", - help="Number of processes to use in pytest (default: auto)", - ) - parser.add_argument("--encref", help=f"REF encoder binary (default:{DEFAULT_ENCODER_REF})") - parser.add_argument("--decref", help=f"REF decoder binary (default:{DEFAULT_DECODER_REF})") - parser.add_argument("--encdut", help=f"DUT encoder binary (default:{DEFAULT_ENCODER_DUT})") - parser.add_argument("--decdut", help=f"DUT decoder binary (default:{DEFAULT_DECODER_DUT})") - - args = parser.parse_args(argv[1:]) - - # check for DUT binaries - if args.encdut: - encdut_path = os.path.realpath(args.encdut) - if not os.path.exists(encdut_path): - sys.exit(f"DUT encoder binary {encdut_path} does not exist.") - else: - encdut_path = DEFAULT_ENCODER_DUT - if args.decdut: - decdut_path = os.path.realpath(args.decdut) - if not os.path.exists(decdut_path): - sys.exit(f"DUT encoder binary {decdut_path} does not exist.") - else: - decdut_path = DEFAULT_DECODER_DUT - if not os.path.exists(encdut_path) or not os.path.exists(decdut_path): - build_dut_binaries() - - if not os.path.exists(REFERENCE_DIR): - # check for REF binaries - if args.encref: - encref_path = os.path.realpath(args.encref) - if not os.path.exists(encref_path): - sys.exit(f"REF encoder binary {encref_path} does not exist.") - else: - encref_path = DEFAULT_ENCODER_REF - if args.decref: - decref_path = os.path.realpath(args.decref) - if not os.path.exists(decref_path): - sys.exit(f"REF encoder binary {decref_path} does not exist.") - else: - decref_path = DEFAULT_DECODER_REF - if not os.path.exists(encref_path) or not os.path.exists(decref_path): - sys.exit("Reference binaries do not exist.") - - # check for test vectors - if not os.path.exists(TEST_VECTOR_DIR): - sys.exit(f"Test vector directory {TEST_VECTOR_DIR} does not exist.") - - # check for references - if os.path.exists(REFERENCE_DIR): - print(f"Using existing references directory {REFERENCE_DIR}") - else: - # create references - print(f"Creating references within the references directory {REFERENCE_DIR}") - create_short_testvectors() - if platform.system() == "Windows": - base_cmd = ["pytest"] - else: - base_cmd = ["python3", "-m", "pytest"] - base_cmd += [ - "scripts/ivas_pytests/tests", - "-n", - args.numprocesses, - "--update_ref", - "1", - "-v", - "--data_system_tests_path", - TEST_VECTOR_DIR, - "--reference_path", - REFERENCE_DIR, - "--dut_base_path", - DUT_BASE_DIR, - "--ref_encoder_path", - encref_path, - "--ref_decoder_path", - decref_path, - "--dut_encoder_path", - encdut_path, - "--dut_decoder_path", - decdut_path, - ] - # work-around in unit tests via environment variable - # TESTVECTOR_PATH_REL_GROUPB: to specify the test vector directory relative to ivas_pytests folder - # TESTVECTOR_PATH_REL_TRUNK: to specify the test vector directory relative to trunk - my_env = os.environ.copy() - my_env["TESTVECTOR_PATH_REL_GROUPB"] = "testv/" - my_env["TESTVECTOR_PATH_REL_TRUNK"] = "/scripts/ivas_pytests/testv/" # leading "/" is important - my_env["CREND_UNIT_TEST_BIN"] = CREND_UNITTEST_REF - print("pytest command line to be executed from project root folder:") - print(" ".join(base_cmd + ["-m", "create_ref"])) - subprocess.run(base_cmd + ["-m", "create_ref"], check=False, env=my_env) - print("pytest command line to be executed from project root folder:") - print(" ".join(base_cmd + ["-m", "create_ref_part2"])) - subprocess.run(base_cmd + ["-m", "create_ref_part2"], check=False, env=my_env) - - if args.create_only: - return - - # run pytest - if platform.system() == "Windows": - cmd = ["pytest"] - else: - cmd = ["python3", "-m", "pytest"] - cmd += [ - "scripts/ivas_pytests/tests", - "-n", - args.numprocesses, - "-v", - "--data_system_tests_path", - TEST_VECTOR_DIR, - "--reference_path", - REFERENCE_DIR, - "--dut_base_path", - DUT_BASE_DIR, - "--dut_encoder_path", - encdut_path, - "--dut_decoder_path", - decdut_path, - "--junit-xml=report-junit.xml", - ] - # print pytest commandline - print("pytest command line to be executed from project root folder:") - print(" ".join(cmd)) - result = subprocess.run(cmd, check=False) - return result.returncode - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/scripts/ivas_pytests/tests/il2mm.py b/scripts/ivas_pytests/tests/il2mm.py deleted file mode 100644 index eb09593bdbed9b7f347c5107f368c4d2d5410a5d..0000000000000000000000000000000000000000 --- a/scripts/ivas_pytests/tests/il2mm.py +++ /dev/null @@ -1,61 +0,0 @@ -""" - (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. -""" - -import os - - -def il2mm(file_in, num_ch, b_delete=True): - """ - Convert interleaved input file to multiple mono output files. - """ - num_bytes_per_sample = 2 - num_bytes_per_frame = num_bytes_per_sample * num_ch - num_bytes_per_channel = os.path.getsize(file_in) / num_ch - - with open(file_in, "rb") as fid_in: - out_path = os.path.splitext(file_in)[0] - for chan in range(num_ch): - file_out = out_path + str(chan + 1) + "ch.raw" - with open(file_out, "wb") as fid_out: - bytes_written = 0 - offset = chan * num_bytes_per_sample - fid_in.seek(offset, 0) - while bytes_written < num_bytes_per_channel: - data = fid_in.read(num_bytes_per_sample) - fid_in.seek(num_bytes_per_frame - num_bytes_per_sample, 1) - written = fid_out.write(bytes(data)) - assert ( - written == num_bytes_per_sample - ), f"Error writing data: {written} != {num_bytes_per_sample}" - bytes_written += num_bytes_per_sample - - # delete interleaved input file - if b_delete: - os.remove(file_in) diff --git a/scripts/ivas_pytests/tests/requirements.txt b/scripts/ivas_pytests/tests/requirements.txt deleted file mode 100644 index 764694dfc08fd19417affb6114620a54eb5f7802..0000000000000000000000000000000000000000 --- a/scripts/ivas_pytests/tests/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -pytest==5.3.5 -pytest-xdist==1.31.0 -scipy==1.5.2 -numpy==1.19.2 diff --git a/scripts/ivas_pytests/tests/system_tests/test_spar_foa_bs_dec_plc.py b/scripts/ivas_pytests/tests/system_tests/test_spar_foa_bs_dec_plc.py deleted file mode 100644 index fa17fd1d802ddec3e6bb05251fc5b12b3686679d..0000000000000000000000000000000000000000 --- a/scripts/ivas_pytests/tests/system_tests/test_spar_foa_bs_dec_plc.py +++ /dev/null @@ -1,180 +0,0 @@ -""" - (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. -""" - -import os -import pytest -import shutil -import errno -import sys - -sys.path.append('scripts/ivas_pytests/') -sys.path.append('scripts/ivas_pytests/tests/') -from il2mm import il2mm -from cmp_custom import cmp_custom -from conftest import EncoderFrontend, DecoderFrontend - -#params -tag_list = ['stvFOA'] -plc_patterns = ['PLperc12mblen5', 'PLperc40mblen50', 'PLperc42mblen2'] -dtx_set = ['0', '1'] -ivas_br_list = ['32000', '64000', '96000', '256000'] -sampling_rate_list = ['48', '32', '16'] -agc_list = [0, 1] - -ch_count_foa = 4 -AbsTol = '3' - - -def check_and_makedir(dir_path): - if not os.path.exists(dir_path): - try: - os.makedirs(dir_path) - except OSError as e: - if e.errno != errno.EEXIST: - raise # raises the error again - - -# assumption: -# - the needed reference bitstreams are created by test_spar_foa_enc_system -# -> reference bitstreams are not any longer created as part of this test -# -> the parameters of this test (except additional parameter plc_pattern) need to be a subset of the parameters in test_spar_foa_enc_system -# -> the reference generation for this test (reference decoder output) needs to be done after completion of test_spar_foa_enc_system -# -> therefore the marker create_ref_part2 -@pytest.mark.create_ref_part2 -@pytest.mark.parametrize("ivas_br", ivas_br_list) -@pytest.mark.parametrize("dtx", dtx_set) -@pytest.mark.parametrize("tag", tag_list) -@pytest.mark.parametrize("plc_pattern", plc_patterns) -@pytest.mark.parametrize("fs", sampling_rate_list) -@pytest.mark.parametrize("agc", agc_list) -def test_spar_foa_plc_system( - dut_decoder_frontend: DecoderFrontend, - data_system_tests_path, - reference_path, - dut_base_path, - ref_decoder_path, - update_ref, - ivas_br, - dtx, - tag, - plc_pattern, - fs, - agc -): - tag = tag + fs + 'c' - - #dec - spar_foa_dec_plc(dut_decoder_frontend, data_system_tests_path, reference_path, dut_base_path, ref_decoder_path, tag, ch_count_foa, fs, ivas_br, dtx, plc_pattern, update_ref, agc) - - -######################################################### -############ test function ############################## -def spar_foa_dec_plc( - decoder_frontend, - test_vector_path, - reference_path, - dut_base_path, - ref_decoder_path, - tag, - ch_count, - sampling_rate, - ivas_br, - dtx, - plc_pattern, - update_ref, - agc -): - - ######### run cmd ##################################### - - tag_out = f"{tag}_ivasbr{ivas_br[:-3]}k_DTX{dtx}" - if agc == 1: - tag_out += '_AGC1' - plc_tag_out = f"{tag_out}_{plc_pattern}" - - dut_out_dir = f"{dut_base_path}/spar_foa_bs/raw/{plc_tag_out}" - ref_out_dir = f"{reference_path}/spar_foa_bs/raw/{plc_tag_out}" - - check_and_makedir(dut_out_dir) - check_and_makedir(ref_out_dir) - - plc_file = f"{test_vector_path}/{plc_pattern}.g192" - ref_in_pkt = f"{reference_path}/spar_foa_bs/pkt/{tag_out}.pkt" - ref_in_pkt_dutenc = f"{reference_path}/spar_foa_bs/pkt/{tag_out}_dutenc.pkt" - - if ref_decoder_path: - ref_decoder = DecoderFrontend(ref_decoder_path, "REF") - - # call REF decoder - ref_decoder.run( - "FOA", - sampling_rate, - ref_in_pkt, - f"{ref_out_dir}/out.raw", - plc_file=plc_file, - ) - - # convert REF interleaved to multi-mono - il2mm(f"{ref_out_dir}/out.raw", ch_count) - - if update_ref == 0: - # call DUT decoder - decoder_frontend.run( - "FOA", - sampling_rate, - ref_in_pkt_dutenc, - f"{dut_out_dir}/out.raw", - plc_file=plc_file, - ) - - il2mm(f"{dut_out_dir}/out.raw", ch_count) - - ######### compare cmd ##################################### - - end_skip_samples = '0' - - test_fail = False - for count in range(ch_count): - ch_id = str(count + 1) - - if cmp_custom( - f"{dut_out_dir}/out{ch_id}ch.raw", - f"{ref_out_dir}/out{ch_id}ch.raw", - "2", - AbsTol, - end_skip_samples - ) != 0: - test_fail = True - - ##File removal## - shutil.rmtree(dut_out_dir, ignore_errors=True) - - ##report failure - assert not test_fail diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6ba72e4aced2ae06f1ae8e004492acceeb07a9ce --- /dev/null +++ b/tests/README.md @@ -0,0 +1,173 @@ +# IVAS tests + +The IVAS tests are using the [pytest](https://docs.pytest.org/) framework. + +## Installing test dependencies + +To use the `pytest` framework, you will need a few Python packages (in addition to Python itself). +As with other Python packages, there are different possibilities to install those packages. +Please chose the option that works best for you. + +`Note`: +The installation of Python is not described, here. +In the following, it is assumed that `Python >= 3.7` is already installed / present. + +### Global install + +```bash +pip install -r tests/requirements.txt +``` + +### User install + +```bash +pip install --user -r tests/requirements.txt +``` + +### Virtual environment install + +```bash +# set up virtual environment +python3 -m venv VENV_NAME +# change to virtual environment +source VENV_NAME/bin/activate +# install required packages +pip install -r tests/requirements.txt +``` + +## Preparing the tests + +`Note:` +Currently, shortened test vectors are used to speed up the testing. +Those shortened test vectors, some with gain adjustment, need to be created, once. + +```bash +# create shortened test vectors +python3 tests/create_short_testvectors.py +``` + +The tests rely on references which need to be generated upfront using reference binaries. +When the reference binaries are named `IVAS_cod_ref(.exe)` and `IVAS_dec_ref(.exe)`, pytest will find and use them. +When the reference binaries are named differently, you need to specify them via the `--ref_encoder_path` and `--ref_decoder_path` options. + +The tests will use the binaries `IVAS_cod(.exe)` and `IVAS_dec(.exe)` for testing. Please make sure that the binaries have been built before running the tests. +When different test binaries are to be used, they can be specified via the `--dut_encoder_path` and `--dut_decoder_path` options. +(DUT: Device Under Test) + +```bash +# create references +# the following binaries need to be present: +# - IVAS_cod(.exe) +# - IVAS_dec(.exe) +# - IVAS_cod_ref(.exe) +# - IVAS_dec_ref(.exe) +# pytest command lines to be executed from project root folder: +pytest tests --update_ref 1 -m create_ref +pytest tests --update_ref 1 -m create_ref_part2 +``` + +## Running the tests + +To run all tests from the tests folder: + +```bash +# pytest command line to be executed from project root folder: +pytest tests +``` + +## Re-running some tests + +When there are test failures, you may want to run, after having fixed the code, only those test cases which had failures. This can be achieved using the `--last-failed` option. + +```bash +# rerun only the tests that failed at the last run +pytest tests --last-failed +``` + +To run a specific test case, you can e.g. pick a test case from the `short test summary info` and use that test case as an argument to `pytest`. E.g. + +```bash +# run a specific test case +pytest tests/test_sba_bs_dec_plc.py::test_sba_plc_system[0-48-PLperc12mblen5-stvFOA-0-32000] +``` + +More ways to select which tests to run: + +```bash +# run all tests within a module +pytest tests/test_sba_bs_dec_plc.py +# run a specific test from a module +pytest tests/test_sba_bs_dec_plc.py::test_sba_plc_system +``` + +## Some pytest hints + +When there a many test failures, you can use the `-x` (or `--exitfirst`) option to stop testing on the first failure. + +Commonly used options like `-n auto` are added to addopts within the [pytest] section in `pytest.ini`. This saves some typing when calling `pytest`. + +The `-v` (or `--verbose`) option is helpful to see what is going on. + +## Custom options + +`Note:` +The custom options are listed as part of the pytest help `pytest -h`. + +```text +--update_ref=UPDATE_REF Indicate whether references shall be updated. + 0: Only DUT processing, no reference generation, references need to be present. + 1: Only reference generation (unconditionally), no DUT processing. + 2: DUT processing, references are generated when not present (not supported by all tests). +--dut_encoder_path=DUT_ENCODER_PATH If specified, use given binary as DUT encoder. +--dut_decoder_path=DUT_DECODER_PATH If specified, use given binary as DUT decoder. +--ref_encoder_path=REF_ENCODER_PATH If specified, use given binary as REF encoder. +--ref_decoder_path=REF_DECODER_PATH If specified, use given binary as REF decoder. +--test_vector_path=TEST_VECTOR_PATH If specified, use given directory as base directory for test vector files. +--reference_path=REFERENCE_PATH If specified, use given directory as base directory for reference files. +--dut_base_path=DUT_BASE_PATH If specified, use given directory as base data directory for dut files. +--param_file=PARAM_FILE If specified, use given param file in test_param_file. +--keep_files By default, the DUT output files of successful tests are deleted. + Use --keep_files to prevent these deletions. +``` + +## Helper scripts + +To help with running the tests during development, two scripts are available in the `tests` folder: + +- prepare_pytests.py +- run_pytests.py + +The envisioned development workflow is: + +```bash +# 1. create a new git branch and switch to the branch +git checkout -b new_branch + +# 2. build the REF binaries (here: example for Linux) +make -j + +# 3. use the binaries to generate the references for future tests +# assumption: you want to test your development against the start of the development +tests/prepare_pytests.py +# Note: the script will use the binaries IVAS_cod and IVAS_dec in case IVAS_cod_ref and IVAS_dec_ref are not present + +# 3a. (optional) store REF binaries in case you want to re-run the reference generation at a later stage +cp IVAS_cod IVAS_cod_ref +cp IVAS_dec IVAS_dec_ref + +# 4. do the development changes +edit ... + +# 5. build the DUT binaries (here: example for Linux) +make -j + +# 6. run the tests +tests/run_pytests.py + +# 7. depending on the test result +# - either go back to 4. +# - or commit and push your changes +``` + +Both scripts allow to restrict the reference generation or the testing to test_param_file tests +with a custom `.prm` file via the `--param_file` option. diff --git a/scripts/ivas_pytests/tests/cmp_custom.py b/tests/cmp_custom.py old mode 100644 new mode 100755 similarity index 62% rename from scripts/ivas_pytests/tests/cmp_custom.py rename to tests/cmp_custom.py index e52d6df46cf6804d012ad797c8a7a0acc379b2f1..ab22bc0ceb9793dc81166f29089fbd00fe75f513 --- a/scripts/ivas_pytests/tests/cmp_custom.py +++ b/tests/cmp_custom.py @@ -1,35 +1,37 @@ #!/usr/bin/env python3 +__copyright__ = \ """ - (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ +__doc__ = \ """ Script to compare samples in 2 PCM files. @@ -87,7 +89,7 @@ class CompareSamples: self.file_1.seek(0) self.file_2.seek(0) - def print_summary(self): + def print_summary(self) -> (int, str): """ Print the summary of the comparison. """ @@ -103,19 +105,18 @@ class CompareSamples: if not self.diff_present: print("Comparison success") print("") - return 0 - else: - print( - f"First unmatched diff ==> {self.first_diff}", - f"at sample num {self.first_diff_sample_num}", - ) - print( - f"MAXIMUM ABS DIFF ==> {self.max_diff} at sample num {self.max_diff_sample_num}" - ) - print("Comparison failed") - print("") - return 1 - return 1 + return 0, "Comparison success" + + # comparison failed + print( + f"First unmatched diff ==> {self.first_diff}", + f"at sample num {self.first_diff_sample_num}", + ) + diff_msg = f"MAXIMUM ABS DIFF ==> {self.max_diff} at sample num {self.max_diff_sample_num}" + print(diff_msg) + print("Comparison failed") + print("") + return 1, f"Comparison failed, {diff_msg}" def compare_next_sample(self): """ @@ -146,7 +147,7 @@ class CompareSamples: def usage(): print(__doc__) - return 1 + return 1, "" def cmp_custom( @@ -155,7 +156,7 @@ def cmp_custom( sample_size_in_bytes_str, tolerance_str, end_samples_to_skip_str="0", -): +) -> (int, str): """ Function to compare the samples in 2 PCM files. """ @@ -189,10 +190,11 @@ def cmp_custom( return cmp_samples.print_summary() -def main(argv): +def main(argv) -> int: if len(argv) < 5: return usage() - return cmp_custom(*argv[1:]) + retval, _reason = cmp_custom(*argv[1:]) + return retval if __name__ == "__main__": diff --git a/scripts/ivas_pytests/conftest.py b/tests/conftest.py similarity index 57% rename from scripts/ivas_pytests/conftest.py rename to tests/conftest.py index 2cc74689afa0fee63306d2cfb8ad2cfc4e41fd31..da1c26e54a3735c670cb498af472529f400d53c2 100644 --- a/scripts/ivas_pytests/conftest.py +++ b/tests/conftest.py @@ -1,31 +1,37 @@ +__copyright__ = \ """ - (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +__doc__ = \ +""" +Pytest customization (configuration and fixtures) for the IVAS codec test suite. """ import logging @@ -35,7 +41,7 @@ from subprocess import run import textwrap from typing import Optional import os - +import testconfig import pytest logger = logging.getLogger(__name__) @@ -54,75 +60,104 @@ def log_dbg_msg(message): @pytest.fixture(scope="session", autouse=True) def rootdir(request): + """ + Return root directory for tests. + """ return str(request.config.rootdir) def pytest_addoption(parser): - parser.addoption("--update_ref", action="store", default="0") - parser.addoption("--p4_CL", action="store") - parser.addoption("--p4cmd_active", action="store", default="0") + parser.addoption( + "--update_ref", + action="store", + help="""Indicate whether references shall be updated. + 0: Only DUT processing, no reference generation, references need to be present. + 1: Only reference generation (unconditionally), no DUT processing. + 2: DUT processing, references are generated when not present (not supported by all tests). + """, + default="0", + ) parser.addoption( "--dut_encoder_path", action="store", - help="If specified, use given binary as DUT encoder." + help="If specified, use given binary as DUT encoder.", ) parser.addoption( "--dut_decoder_path", action="store", - help="If specified, use given binary as DUT decoder." + help="If specified, use given binary as DUT decoder.", ) parser.addoption( "--ref_encoder_path", action="store", - help="If specified, use given binary as REF encoder." + help="If specified, use given binary as REF encoder.", ) parser.addoption( "--ref_decoder_path", action="store", - help="If specified, use given binary as REF decoder." + help="If specified, use given binary as REF decoder.", ) - # TODO: rename to test_vector_path parser.addoption( - "--data_system_tests_path", + "--test_vector_path", action="store", - help="If specified, use given directory as base data directory for system tests." + help="If specified, use given directory as base directory for test vector files.", ) parser.addoption( "--reference_path", action="store", - help="If specified, use given directory as base directory for reference files." + help="If specified, use given directory as base directory for reference files.", ) parser.addoption( "--dut_base_path", action="store", - help="If specified, use given directory as base data directory for dut files." + help="If specified, use given directory as base data directory for dut files.", ) + parser.addoption( + "--param_file", + action="store", + help="If specified, use given param file in test_param_file.", + ) -@pytest.fixture(scope="session", autouse=True) -def update_ref(request): - return int(request.config.getoption("--update_ref")) + parser.addoption( + "--keep_files", + action="store_true", + help="By default, the DUT output files of successful tests are deleted." + " Use --keep_files to prevent these deletions.", + ) @pytest.fixture(scope="session", autouse=True) -def p4_CL(request): - return request.config.option.p4_CL +def update_ref(request): + """ + Return indication whether references shall be updated. + 0: Only DUT processing, no reference generation. + 1: Only reference generation (unconditionally), no DUT processing. + 2: DUT processing, references are generated when not present. + """ + return int(request.config.getoption("--update_ref")) -@pytest.fixture(scope="session", autouse=True) -def p4cmd_active(request): - return int(request.config.getoption("--p4cmd_active")) +@pytest.fixture(scope="session") +def keep_files(request) -> bool: + """ + Return indication to not delete DUT output files. + """ + return request.config.option.keep_files @pytest.fixture(scope="session") -def dut_encoder_path(request) -> Path: +def dut_encoder_path(request) -> str: + """ + Return path of DUT encoder binary. + """ if request.config.option.dut_encoder_path: return request.config.option.dut_encoder_path @@ -130,15 +165,16 @@ def dut_encoder_path(request) -> Path: system = platform.system() if system == "Windows": - path = here.joinpath("../../IVAS_cod.exe") + path = here.joinpath("../IVAS_cod.exe") elif system in ["Darwin", "Linux"]: - path = here.joinpath("../../IVAS_cod") + path = here.joinpath("../IVAS_cod") else: raise ValueError(f'Wrong system "{system}"!') path = str(path.resolve()) - assert os.path.isfile(path) + if not os.path.isfile(path): + pytest.exit(f"\nDUT encoder binary {path} not found!\n!") return path @@ -163,6 +199,7 @@ class EncoderFrontend: agc_op: Optional[int] = None, bypass_mode: Optional[int] = None, quiet_mode: Optional[bool] = True, + add_option_list: Optional[list] = None, ) -> None: command = [self._path] @@ -185,6 +222,9 @@ class EncoderFrontend: if quiet_mode: command.extend(["-q"]) + if add_option_list is not None: + command.extend(add_option_list) + # add mandatory parameters command += [ str(bitrate), @@ -221,6 +261,9 @@ class EncoderFrontend: @pytest.fixture(scope="function") def dut_encoder_frontend(dut_encoder_path) -> EncoderFrontend: + """ + Return a :class:`conftest.EncoderFrontend` instance as DUT for the test session. + """ encoder = EncoderFrontend(dut_encoder_path, "DUT") yield encoder @@ -229,34 +272,40 @@ def dut_encoder_frontend(dut_encoder_path) -> EncoderFrontend: @pytest.fixture(scope="session") -def ref_encoder_path(request) -> Path: +def ref_encoder_path(request) -> str: + """ + Return path of REF encoder binary. + """ if request.config.option.ref_encoder_path: return request.config.option.ref_encoder_path - update_ref = int(request.config.getoption("--update_ref")) - if not update_ref: + if request.config.option.update_ref == "0": return None - # assume default encoder when update_ref is selected, but no ref_encoder_path is specified + # assume specifically named encoder when update_ref is selected, but no ref_encoder_path is specified here = Path(__file__).parent.resolve() system = platform.system() if system == "Windows": - path = here.joinpath("../../IVAS_cod.exe") + path = here.joinpath("../IVAS_cod_ref.exe") elif system in ["Darwin", "Linux"]: - path = here.joinpath("../../IVAS_cod") + path = here.joinpath("../IVAS_cod_ref") else: raise ValueError(f'Wrong system "{system}"!') path = str(path.resolve()) - assert os.path.isfile(path) + if not os.path.isfile(path): + pytest.exit(f"\nREF encoder binary {path} not found!\n!") return path @pytest.fixture(scope="session") -def dut_decoder_path(request) -> Path: +def dut_decoder_path(request) -> str: + """ + Return path of DUT decoder binary. + """ if request.config.option.dut_decoder_path: return request.config.option.dut_decoder_path @@ -264,15 +313,16 @@ def dut_decoder_path(request) -> Path: system = platform.system() if system == "Windows": - path = here.joinpath("../../IVAS_dec.exe") + path = here.joinpath("../IVAS_dec.exe") elif system in ["Darwin", "Linux"]: - path = here.joinpath("../../IVAS_dec") + path = here.joinpath("../IVAS_dec") else: raise ValueError(f'Wrong system "{system}"!') path = str(path.resolve()) - assert os.path.isfile(path) + if not os.path.isfile(path): + pytest.exit(f"\nDUT decoder binary {path} not found!\n!") return path @@ -293,6 +343,7 @@ class DecoderFrontend: output_path: Path, quiet_mode: Optional[bool] = True, plc_file: Optional[Path] = None, + add_option_list: Optional[list] = None, ) -> None: command = [self._path] @@ -303,9 +354,14 @@ class DecoderFrontend: if plc_file is not None: command.extend(["-fec", str(plc_file)]) + if add_option_list is not None: + command.extend(add_option_list) + # add mandatory parameters + # output_config is mandatory for IVAS; EVS does not have this parameter, indicated by "" + if output_config != "": + command += [output_config] command += [ - output_config, str(output_sampling_rate), str(input_bitstream_path), str(output_path), @@ -339,6 +395,9 @@ class DecoderFrontend: @pytest.fixture(scope="function") def dut_decoder_frontend(dut_decoder_path) -> DecoderFrontend: + """ + Return a :class:`conftest.DecoderFrontend` instance as DUT for the test session. + """ decoder = DecoderFrontend(dut_decoder_path, "DUT") yield decoder @@ -347,40 +406,46 @@ def dut_decoder_frontend(dut_decoder_path) -> DecoderFrontend: @pytest.fixture(scope="session") -def ref_decoder_path(request) -> Path: +def ref_decoder_path(request) -> str: + """ + Return path of REF decoder binary. + """ if request.config.option.ref_decoder_path: return request.config.option.ref_decoder_path - update_ref = int(request.config.getoption("--update_ref")) - if not update_ref: + if request.config.option.update_ref == "0": return None - # assume default decoder when update_ref is selected, but no ref_decoder_path is specified + # assume specifically named decoder when update_ref is selected, but no ref_decoder_path is specified here = Path(__file__).parent.resolve() system = platform.system() if system == "Windows": - path = here.joinpath("../../IVAS_dec.exe") + path = here.joinpath("../IVAS_dec_ref.exe") elif system in ["Darwin", "Linux"]: - path = here.joinpath("../../IVAS_dec") + path = here.joinpath("../IVAS_dec_ref") else: raise ValueError(f'Wrong system "{system}"!') path = str(path.resolve()) - assert os.path.isfile(path) + if not os.path.isfile(path): + pytest.exit(f"\nREF decoder binary {path} not found!\n!") return path @pytest.fixture(scope="session") -def data_system_tests_path(request) -> Path: - if request.config.option.data_system_tests_path: - return request.config.option.data_system_tests_path +def test_vector_path(request) -> str: + """ + Return base directory of test vector files. + """ + if request.config.option.test_vector_path: + return request.config.option.test_vector_path here = Path(__file__).parent.resolve() - path = here.joinpath("testv") + path = here.joinpath("../scripts/testv") path = str(path.resolve()) @@ -388,7 +453,10 @@ def data_system_tests_path(request) -> Path: @pytest.fixture(scope="session") -def reference_path(request) -> Path: +def reference_path(request) -> str: + """ + Return base directory of reference files. + """ if request.config.option.reference_path: return request.config.option.reference_path @@ -398,11 +466,18 @@ def reference_path(request) -> Path: path = str(path.resolve()) + if request.config.option.update_ref == "0": + if not os.path.isdir(path): + pytest.exit(f"\nREF path {path} not found!\nPlease generate the references, first!\n!") + return path @pytest.fixture(scope="session") -def dut_base_path(request) -> Path: +def dut_base_path(request) -> str: + """ + Return base data directory for dut files. + """ if request.config.option.dut_base_path: return request.config.option.dut_base_path @@ -425,3 +500,5 @@ def pytest_configure(config): config.addinivalue_line( "markers", "create_ref_part2: reference creation test that depends on create_ref references" ) + if config.option.param_file: + testconfig.PARAM_FILE = config.option.param_file diff --git a/tests/create_short_testvectors.py b/tests/create_short_testvectors.py new file mode 100755 index 0000000000000000000000000000000000000000..13afe8a422f291c2b27d896f06e0410e02754628 --- /dev/null +++ b/tests/create_short_testvectors.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +__copyright__ = \ +""" +(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +__doc__ = \ +""" +Create short (5sec) testvectors. +""" + +import sys +from pathlib import Path +from cut_pcm import cut_samples + +HERE = Path(__file__).parent.resolve() +TEST_VECTOR_DIR = str(HERE.joinpath("../scripts/testv").resolve()) + +NUM_CHANNELS = "4" # currently only FOA +CUT_FROM = "0.0" +CUT_LEN = "5.0" + + +def create_short_testvectors(): + for fs in ['48', '32', '16']: + in_file = f"{TEST_VECTOR_DIR}/stvFOA{fs}c.pcm" + cut_gain = "1.0" + cut_file = f"{TEST_VECTOR_DIR}/stvFOA{fs}c_cut.pcm" + cut_samples(in_file, cut_file, NUM_CHANNELS, fs + "000", CUT_FROM, CUT_LEN, cut_gain) + cut_gain = "16.0" + cut_file = f"{TEST_VECTOR_DIR}/stvFOA{fs}c_cut_{cut_gain}.pcm" + cut_samples(in_file, cut_file, NUM_CHANNELS, fs + "000", CUT_FROM, CUT_LEN, cut_gain) + + +if __name__ == "__main__": + sys.exit(create_short_testvectors()) diff --git a/scripts/ivas_pytests/tests/cut_pcm.py b/tests/cut_pcm.py similarity index 99% rename from scripts/ivas_pytests/tests/cut_pcm.py rename to tests/cut_pcm.py index 62af257a7351424850b5aa3185fe22d2b8ccaccc..938cb6fc43549aba239304e980f1ddcd54f6058a 100755 --- a/scripts/ivas_pytests/tests/cut_pcm.py +++ b/tests/cut_pcm.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -__license__ = """ +__copyright__ = \ +""" (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, @@ -30,7 +31,8 @@ accordance with the laws of the Federal Republic of Germany excluding its confli the United Nations Convention on Contracts on the International Sales of Goods. """ -__doc__ = """ +__doc__ = \ +""" Script to cut samples from a 16-bit PCM file. USAGE : cut_pcm.py in_file_pcm out_file_pcm num_channels sample_rate start duration [gain] diff --git a/tests/prepare_pytests.py b/tests/prepare_pytests.py new file mode 100755 index 0000000000000000000000000000000000000000..d1f7495f0712bfa0cbb033f454262867cbdcb6c0 --- /dev/null +++ b/tests/prepare_pytests.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 + +__copyright__ = """ +(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +__doc__ = """ +Script to prepare the pytest tests. +""" + +import os +import sys +import argparse +import subprocess +import platform + +from pathlib import Path +from create_short_testvectors import create_short_testvectors + +BIN_EXT = ".exe" if platform.system() == "Windows" else "" +HERE = Path(__file__).parent.resolve() +DEFAULT_ENCODER_DUT = str(HERE.joinpath(f"../IVAS_cod{BIN_EXT}").resolve()) +DEFAULT_DECODER_DUT = str(HERE.joinpath(f"../IVAS_dec{BIN_EXT}").resolve()) +DEFAULT_ENCODER_REF = str(HERE.joinpath(f"../IVAS_cod_ref{BIN_EXT}").resolve()) +DEFAULT_DECODER_REF = str(HERE.joinpath(f"../IVAS_dec_ref{BIN_EXT}").resolve()) +REFERENCE_DIR = str(HERE.joinpath("ref").resolve()) + + +def main(argv): + """ + Prepare the pytest tests. + """ + # check for python >= 3.7 + if sys.version_info[0] < 3 or sys.version_info[1] < 7: + sys.exit("This script is written for Python >= 3.7. Found: " + platform.python_version()) + + parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + "--numprocesses", + action="store", + default="auto", + help="Number of processes to use in pytest (default: auto)", + ) + parser.add_argument( + "--param_file", + action="store", + help="Restrict reference generation to test_param_file with specified param file.", + ) + + args = parser.parse_args(argv[1:]) + + use_dut_binaries = False + + # check for existing references + if os.path.exists(REFERENCE_DIR): + sys.exit( + f"Found existing references directory {REFERENCE_DIR}.\n" + "Please delete this directory if you want the references to be recreated." + ) + + # check for DUT binaries + if not os.path.exists(DEFAULT_ENCODER_DUT) or not os.path.exists(DEFAULT_DECODER_DUT): + sys.exit( + f"Need DUT binaries {DEFAULT_ENCODER_DUT} and {DEFAULT_DECODER_DUT}.\n" + "Please create the binaries." + ) + + # check for REF binaries + if not os.path.exists(DEFAULT_ENCODER_REF) or not os.path.exists(DEFAULT_DECODER_REF): + print(f"REF binaries {DEFAULT_ENCODER_REF} and {DEFAULT_DECODER_REF} not found.") + print("DUT binaries will be used for reference generation.") + use_dut_binaries = True + + # create references + print(f"Creating references within the references directory {REFERENCE_DIR}.") + create_short_testvectors() + if platform.system() == "Windows": + base_cmd = ["pytest"] + else: + base_cmd = ["python3", "-m", "pytest"] + if args.param_file: + base_cmd += ["tests/test_param_file.py", "--param_file", args.param_file] + else: + base_cmd += ["tests"] + base_cmd += [ + "-n", + args.numprocesses, + "--update_ref", + "1", + ] + if use_dut_binaries: + base_cmd += [ + "--ref_encoder_path", + DEFAULT_ENCODER_DUT, + "--ref_decoder_path", + DEFAULT_DECODER_DUT, + ] + + result = subprocess.run(base_cmd + ["-m", "create_ref"], check=False) + if not args.param_file: + result = subprocess.run(base_cmd + ["-m", "create_ref_part2"], check=False) + return result.returncode + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..2eb090f4fb56da7587e0a3eb19b7e287870361c1 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,4 @@ +pytest>=5.3.5 +pytest-xdist>=1.31.0 +scipy>=1.5.2 +numpy>=1.19.2 diff --git a/tests/run_pytests.py b/tests/run_pytests.py new file mode 100755 index 0000000000000000000000000000000000000000..1fc66147622b20faf1359eed855f53b937f4c7c6 --- /dev/null +++ b/tests/run_pytests.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 + +__copyright__ = """ +(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +__doc__ = """ +Script to run the pytest tests. + +Test prerequisites are checked for and check failures are reported. +When prerequisites are met, the pytest test is executed. +""" + +import os +import sys +import argparse +import subprocess +import platform +from pathlib import Path + +BIN_EXT = ".exe" if platform.system() == "Windows" else "" +HERE = Path(__file__).parent.resolve() +DEFAULT_ENCODER_DUT = str(HERE.joinpath(f"../IVAS_cod{BIN_EXT}").resolve()) +DEFAULT_DECODER_DUT = str(HERE.joinpath(f"../IVAS_dec{BIN_EXT}").resolve()) +REFERENCE_DIR = str(HERE.joinpath("ref").resolve()) + + +def main(argv): + """ + Run the pytest tests. + """ + # check for python >= 3.7 + if sys.version_info[0] < 3 or sys.version_info[1] < 7: + sys.exit("This script is written for Python >= 3.7. Found: " + platform.python_version()) + + parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + "--numprocesses", + action="store", + default="auto", + help="Number of processes to use in pytest (default: auto)", + ) + parser.add_argument( + "--param_file", + action="store", + help="Restrict test run to test_param_file with specified param file.", + ) + + args = parser.parse_args(argv[1:]) + + # check for references + if not os.path.exists(REFERENCE_DIR): + sys.exit( + f"References directory {REFERENCE_DIR} not found.\nPlease create the references." + ) + + # check for DUT binaries + if not os.path.exists(DEFAULT_ENCODER_DUT) or not os.path.exists(DEFAULT_DECODER_DUT): + sys.exit( + f"Need DUT binaries {DEFAULT_ENCODER_DUT} and {DEFAULT_DECODER_DUT}.\n" + "Please create the binaries." + ) + + # run pytest + if platform.system() == "Windows": + cmd = ["pytest"] + else: + cmd = ["python3", "-m", "pytest"] + if args.param_file: + cmd += ["tests/test_param_file.py", "--param_file", args.param_file] + else: + cmd += ["tests"] + cmd += ["-n", args.numprocesses] + + result = subprocess.run(cmd, check=False) + return result.returncode + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tests/test_param_file.py b/tests/test_param_file.py new file mode 100644 index 0000000000000000000000000000000000000000..730acd5c00ce6d065ab2002948b451e45b6067db --- /dev/null +++ b/tests/test_param_file.py @@ -0,0 +1,460 @@ +__copyright__ = \ +""" +(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +__doc__ = \ +""" +Execute tests specified via a parameter file. +""" + +import os +import errno +import platform +from subprocess import run +import pytest +from cmp_custom import cmp_custom +from conftest import EncoderFrontend, DecoderFrontend +from testconfig import PARAM_FILE + + +VALID_DEC_OUTPUT_CONF = [ + "MONO", + "STEREO", + "5_1", + "7_1", + "5_1_2", + "5_1_4", + "7_1_4", + "FOA", + "HOA2", + "HOA3", + "BINAURAL", + "BINAURAL_ROOM", + "EXT", +] + +param_file_test_dict = {} +with open(PARAM_FILE, "r", encoding="UTF-8") as fp: + data = fp.read() + blocks = data.split("\n\n") + for block in blocks: + tag = "" + enc_opts = "" + dec_opts = "" + sim_opts = "" + for line in block.split("\n"): + if line.startswith("// "): + tag = line[3:] + if line.startswith("../IVAS_cod "): + enc_opts = line[12:] + if line.startswith("../IVAS_dec "): + dec_opts = line[12:] + if line.startswith("networkSimulator_g192 "): + sim_opts = line[22:] + if tag == "" or enc_opts == "" or dec_opts == "": + # no complete parameter set + continue + if tag in param_file_test_dict: + print("non-unique tag found - ignoring new entry") + continue + param_file_test_dict[tag] = (enc_opts, dec_opts, sim_opts) + + +def check_and_makedir(dir_path): + if not os.path.exists(dir_path): + try: + os.makedirs(dir_path) + except OSError as e: + if e.errno != errno.EEXIST: + raise # raises the error again + + +def convert_test_string_to_tag(test_string): + """ + Convert a test string (i.e. the test tag from the parameter file) to a tag string. + Example: + in: "DFT stereo at 13.2 kbps, 16kHz in, 16kHz out, DTX on, random FEC at 5%" + out: "DFT_stereo_at_13_2_kbps_16kHz_in_16kHz_out_DTX_on_random_FEC_at_5_" + """ + # replace certain characters by "_" or remove them + tag_str = "" + replace_chars = " %.-()" + remove_chars = "," + for char in test_string: + if char in replace_chars: + tag_str += "_" + elif char not in remove_chars: + tag_str += char + # replace double underscore by single one + tag_str = "_".join(tag_str.split("__")) + return tag_str + + +@pytest.mark.create_ref +@pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys())) +def test_param_file_tests( + dut_encoder_frontend: EncoderFrontend, + dut_decoder_frontend: DecoderFrontend, + ref_encoder_path, + ref_decoder_path, + reference_path, + dut_base_path, + test_vector_path, + update_ref, + rootdir, + keep_files, + test_tag, +): + enc_opts, dec_opts, sim_opts = param_file_test_dict[test_tag] + + tag_str = convert_test_string_to_tag(test_tag) + + # evaluate encoder options + enc_split = enc_opts.split() + assert len(enc_split) >= 4 + + # replace "testv/" by test vector path + enc_split = [ + x.replace("testv", f"{test_vector_path}", 1) if x.startswith("testv/") else x + for x in enc_split + ] + + bitstream_file = enc_split.pop() + testv_file = enc_split.pop() + sampling_rate = int(enc_split.pop()) + bitrate = enc_split.pop() + + # bitrate can be a filename: remove leading "../" + if bitrate.startswith("../"): + bitrate = bitrate[3:] + + testv_base = testv_file.split("/")[-1] + if testv_base.endswith(".pcm"): + testv_base = testv_base[:-4] + + assert bitstream_file == "bit" + # in the parameter file, only "bit" is used as bitstream file name + # -> construct bitstream filename + bitstream_file = f"{testv_base}_{tag_str}.192" + + encode( + dut_encoder_frontend, + ref_encoder_path, + reference_path, + dut_base_path, + bitrate, + sampling_rate, + testv_file, + bitstream_file, + enc_split, + update_ref, + ) + + # check for networkSimulator_g192 command line + if sim_opts != "": + sim_split = sim_opts.split() + assert len(sim_split) == 6, "networkSimulator_g192 binary expects 6 parameters" + # [sim_profile, sim_input, sim_output, sim_trace, sim_nFPP, sim_offset] = sim_split + if sim_split[0].startswith(("../")): + # remove leading "../" + sim_split[0] = sim_split[0][3:] + assert sim_split[1] == "bit" + # in the parameter file, only "bit" is used as bitstream file name + # -> re-use bitstream filename from encoder call + sim_split[1] = bitstream_file + assert sim_split[2] == "netsimoutput" + # in the parameter file, only "netsimoutput" is used as netsim output file name + # -> construct netsim output file name + netsim_outfile = f"{testv_base}_{tag_str}.netsimout" + sim_split[2] = netsim_outfile + assert sim_split[3] == "tracefile_sim" + # in the parameter file, only "tracefile_sim" is used as trace output file name + # -> construct trace output file name + netsim_trace_outfile = f"{testv_base}_{tag_str}.netsimtrace" + sim_split[3] = netsim_trace_outfile + simulate( + reference_path, + dut_base_path, + sim_split, + update_ref, + rootdir, + ) + + # evaluate decoder options + dec_split = dec_opts.split() + assert len(dec_split) >= 3 + + # replace "testv/" by test vector path + dec_split = [ + x.replace("testv", f"{test_vector_path}", 1) if x.startswith("testv/") else x + for x in dec_split + ] + # remove leading "../" + dec_split = [x[3:] if x.startswith("../") else x for x in dec_split] + + output_file = dec_split.pop() + bitstream_file_dec = dec_split.pop() + sampling_rate = int(dec_split.pop()) + if len(dec_split) > 0: + output_config = dec_split.pop() + if output_config.upper() not in VALID_DEC_OUTPUT_CONF: + if not output_config.endswith(".txt"): + # must be EVS tests with additional parameters - put param back + dec_split.append(output_config) + output_config = "" + else: + output_config = "" + + output_config_name = output_config + if "/" in output_config: + # the output config is a file + output_config_name = os.path.splitext(os.path.basename(output_config))[0] + + tracefile_dec = "" + if sim_opts != "": + assert bitstream_file_dec == "netsimoutput" + # in the parameter file, only "netsimoutput" is used as bitstream file name + # -> re-use netsim_outfile + bitstream_file = netsim_outfile + tracefile_dec = f"{testv_base}_{tag_str}.dectrace" + else: + assert bitstream_file_dec == "bit" + # in the parameter file, only "bit" is used as bitstream file name + # -> re-use bitstream filename from encoder call + + # the output file is not the real output filename + # -> construct output filename + if output_config != "": + output_file = f"{testv_base}_{tag_str}.dec.{output_config_name}.pcm" + else: + # EVS decoder command lines do not have an output_config: use "MONO" in the output filename + output_file = f"{testv_base}_{tag_str}.dec.MONO.pcm" + + decode( + dut_decoder_frontend, + ref_decoder_path, + reference_path, + dut_base_path, + output_config, + sampling_rate, + bitstream_file, + output_file, + dec_split, + update_ref, + tracefile_dec, + ) + + # compare + if update_ref in [0, 2]: + compare( + f"{dut_base_path}/param_file/dec/{output_file}", + f"{reference_path}/param_file/dec/{output_file}", + ) + + # remove DUT output files when test result is OK (to save disk space) + if not keep_files: + os.remove(f"{dut_base_path}/param_file/enc/{bitstream_file}") + os.remove(f"{dut_base_path}/param_file/dec/{output_file}") + if sim_opts != "": + os.remove(f"{dut_base_path}/param_file/enc/{testv_base}_{tag_str}.192") + os.remove(f"{dut_base_path}/param_file/enc/{netsim_trace_outfile}") + os.remove(f"{dut_base_path}/param_file/dec/{tracefile_dec}") + + +def encode( + encoder_frontend, + ref_encoder_path, + reference_path, + dut_base_path, + bitrate, + sampling_rate, + testv_file, + bitstream_file, + enc_opts_list, + update_ref, +): + """ + Call REF and/or DUT encoder. + """ + # directories + dut_out_dir = f"{dut_base_path}/param_file/enc" + ref_out_dir = f"{reference_path}/param_file/enc" + + ref_out_file = f"{ref_out_dir}/{bitstream_file}" + dut_out_file = f"{dut_out_dir}/{bitstream_file}" + + if update_ref == 1 or update_ref == 2 and not os.path.exists(ref_out_file): + check_and_makedir(ref_out_dir) + # call REF encoder + assert ref_encoder_path + ref_encoder = EncoderFrontend(ref_encoder_path, "REF") + ref_encoder.run( + bitrate, + sampling_rate, + testv_file, + ref_out_file, + add_option_list=enc_opts_list, + ) + + if update_ref in [0, 2]: + check_and_makedir(dut_out_dir) + # call DUT encoder + encoder_frontend.run( + bitrate, + sampling_rate, + testv_file, + dut_out_file, + add_option_list=enc_opts_list, + ) + + +def simulate( + reference_path, + dut_base_path, + sim_opts_list, + update_ref, + rootdir, +): + """ + Call network simulator on REF and/or DUT encoder output. + """ + # directories + dut_out_dir = f"{dut_base_path}/param_file/enc" + ref_out_dir = f"{reference_path}/param_file/enc" + + netsim_infile = sim_opts_list[1] + netsim_outfile = sim_opts_list[2] + netsim_tracefile = sim_opts_list[3] + ref_out_file = f"{ref_out_dir}/{netsim_outfile}" + + if platform.system() == "Windows": + netsim = [os.path.join(rootdir, "scripts", "tools", "Win32", "networkSimulator_g192.exe")] + elif platform.system() == "Linux": + # there is no Linux binary available -> use the Win32 binary via wine + netsim = [ + "wine", + os.path.join(rootdir, "scripts", "tools", "Win32", "networkSimulator_g192.exe"), + ] + elif platform.system() == "Darwin": + netsim = [os.path.join(rootdir, "scripts", "tools", "Darwin", "networkSimulator_g192")] + + if update_ref == 1 or update_ref == 2 and not os.path.exists(ref_out_file): + # call network simulator on REF encoder output + cmd_opts = sim_opts_list + cmd_opts[1] = f"{ref_out_dir}/{netsim_infile}" + cmd_opts[2] = f"{ref_out_dir}/{netsim_outfile}" # ref_out_file + cmd_opts[3] = f"{ref_out_dir}/{netsim_tracefile}" + run(netsim + cmd_opts, check=False) + + if update_ref in [0, 2]: + # call network simulator on DUT encoder output + cmd_opts = sim_opts_list + cmd_opts[1] = f"{dut_out_dir}/{netsim_infile}" + cmd_opts[2] = f"{dut_out_dir}/{netsim_outfile}" # dut_out_file + cmd_opts[3] = f"{dut_out_dir}/{netsim_tracefile}" + run(netsim + cmd_opts, check=False) + + +def decode( + decoder_frontend, + ref_decoder_path, + reference_path, + dut_base_path, + output_config, + sampling_rate, + bitstream_file, + output_file, + dec_opts_list, + update_ref, + tracefile_dec, +): + """ + Call REF and/or DUT decoder. + """ + # directories + dut_out_dir = f"{dut_base_path}/param_file/dec" + ref_out_dir = f"{reference_path}/param_file/dec" + + dut_in_file = f"{dut_base_path}/param_file/enc/{bitstream_file}" + ref_in_file = f"{reference_path}/param_file/enc/{bitstream_file}" + dut_out_file = f"{dut_out_dir}/{output_file}" + ref_out_file = f"{ref_out_dir}/{output_file}" + + if update_ref == 1 or update_ref == 2 and not os.path.exists(ref_out_file): + check_and_makedir(ref_out_dir) + add_option_list = dec_opts_list + if tracefile_dec != "": + add_option_list = [ + x if x != "tracefile_dec" else f"{ref_out_dir}/{tracefile_dec}" + for x in dec_opts_list + ] + # call REF decoder + assert ref_decoder_path + ref_decoder = DecoderFrontend(ref_decoder_path, "REF") + ref_decoder.run( + output_config, + sampling_rate, + ref_in_file, + ref_out_file, + add_option_list=add_option_list, + ) + + if update_ref in [0, 2]: + check_and_makedir(dut_out_dir) + add_option_list = dec_opts_list + if tracefile_dec != "": + add_option_list = [ + x if x != "tracefile_dec" else f"{dut_out_dir}/{tracefile_dec}" + for x in dec_opts_list + ] + # call DUT decoder + decoder_frontend.run( + output_config, + sampling_rate, + dut_in_file, + dut_out_file, + add_option_list=add_option_list, + ) + + +def compare( + pcm_file_1, + pcm_file_2, +): + """ + Compare two PCM files. + Currently, both PCM files are treated like mono files. + This is just fine when checking for bit-exactness. + More advanced comparisons are possible and might come with a future update. + """ + sample_size = "2" # 16-bit samples + tolerance = "0" # zero tolerance for BE testing + cmp_result, reason = cmp_custom(pcm_file_1, pcm_file_2, sample_size, tolerance) + assert cmp_result == 0, reason diff --git a/tests/test_sba_bs_dec_plc.py b/tests/test_sba_bs_dec_plc.py new file mode 100644 index 0000000000000000000000000000000000000000..58be07ec3394a92e92d8ccd63e67794db1390f07 --- /dev/null +++ b/tests/test_sba_bs_dec_plc.py @@ -0,0 +1,189 @@ +__copyright__ = \ +""" +(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +__doc__ = \ +""" +Execute SBA decoder tests using different PLC patterns. +""" + +import os +import errno +import pytest + +from cmp_custom import cmp_custom +from conftest import DecoderFrontend + +#params +tag_list = ['stvFOA'] +plc_patterns = ['PLperc12mblen5', 'PLperc40mblen50', 'PLperc42mblen2'] +dtx_set = ['0', '1'] +ivas_br_list = ['32000', '64000', '96000', '256000'] +sampling_rate_list = ['48', '32', '16'] +agc_list = [0, 1] + +AbsTol = '3' + + +def check_and_makedir(dir_path): + if not os.path.exists(dir_path): + try: + os.makedirs(dir_path) + except OSError as e: + if e.errno != errno.EEXIST: + raise # raises the error again + + +# assumption: +# - the needed reference bitstreams are created by test_sba_enc_system +# -> reference bitstreams are not any longer created as part of this test +# -> the parameters of this test (except additional parameter plc_pattern) need to be a subset of the parameters in test_sba_enc_system +# -> the reference generation for this test (reference decoder output) needs to be done after completion of test_sba_enc_system +# -> therefore the marker create_ref_part2 +@pytest.mark.create_ref_part2 +@pytest.mark.parametrize("ivas_br", ivas_br_list) +@pytest.mark.parametrize("dtx", dtx_set) +@pytest.mark.parametrize("tag", tag_list) +@pytest.mark.parametrize("plc_pattern", plc_patterns) +@pytest.mark.parametrize("fs", sampling_rate_list) +@pytest.mark.parametrize("agc", agc_list) +def test_sba_plc_system( + dut_decoder_frontend: DecoderFrontend, + test_vector_path, + reference_path, + dut_base_path, + ref_decoder_path, + update_ref, + keep_files, + ivas_br, + dtx, + tag, + plc_pattern, + fs, + agc +): + tag = tag + fs + 'c' + + #dec + sba_dec_plc( + dut_decoder_frontend, + test_vector_path, + reference_path, + dut_base_path, + ref_decoder_path, + tag, + fs, + ivas_br, + dtx, + plc_pattern, + update_ref, + agc, + keep_files, + ) + + +######################################################### +############ test function ############################## +def sba_dec_plc( + decoder_frontend, + test_vector_path, + reference_path, + dut_base_path, + ref_decoder_path, + tag, + sampling_rate, + ivas_br, + dtx, + plc_pattern, + update_ref, + agc, + keep_files, +): + + ######### run cmd ##################################### + + tag_out = f"{tag}_ivasbr{ivas_br[:-3]}k_DTX{dtx}" + if agc == 1: + tag_out += '_AGC1' + plc_tag_out = f"{tag_out}_{plc_pattern}" + + dut_out_dir = f"{dut_base_path}/sba_bs/raw" + ref_out_dir = f"{reference_path}/sba_bs/raw" + + check_and_makedir(dut_out_dir) + check_and_makedir(ref_out_dir) + + plc_file = f"{test_vector_path}/{plc_pattern}.g192" + ref_in_pkt = f"{reference_path}/sba_bs/pkt/{tag_out}.pkt" + ref_in_pkt_dutenc = f"{reference_path}/sba_bs/pkt/{tag_out}_dutenc.pkt" + + dut_out_raw = f"{dut_out_dir}/{plc_tag_out}.raw" + ref_out_raw = f"{ref_out_dir}/{plc_tag_out}.raw" + + if ref_decoder_path: + ref_decoder = DecoderFrontend(ref_decoder_path, "REF") + + # call REF decoder + ref_decoder.run( + "FOA", + sampling_rate, + ref_in_pkt, + ref_out_raw, + plc_file=plc_file, + ) + + if update_ref == 0: + # call DUT decoder + decoder_frontend.run( + "FOA", + sampling_rate, + ref_in_pkt_dutenc, + dut_out_raw, + plc_file=plc_file, + ) + + ######### compare cmd ##################################### + + end_skip_samples = '0' + + cmp_result, reason = cmp_custom( + dut_out_raw, + ref_out_raw, + "2", + AbsTol, + end_skip_samples + ) + + # report compare result + assert cmp_result == 0, reason + + # remove DUT output files when test result is OK (to save disk space) + if not keep_files: + os.remove(dut_out_raw) diff --git a/scripts/ivas_pytests/tests/system_tests/test_spar_foa_bs_enc.py b/tests/test_sba_bs_enc.py similarity index 71% rename from scripts/ivas_pytests/tests/system_tests/test_spar_foa_bs_enc.py rename to tests/test_sba_bs_enc.py index df5d018ce50106deae5cbb62aff1c81ec42ade6a..39991640205b894d6d122563aed720984e49650d 100644 --- a/scripts/ivas_pytests/tests/system_tests/test_spar_foa_bs_enc.py +++ b/tests/test_sba_bs_enc.py @@ -1,47 +1,44 @@ +__copyright__ = \ """ - (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ +__doc__ = \ """ Test file to run C encoder and decoder code. The outputs are compared with C generated references. """ import os -import pytest import errno -import shutil -import sys +import pytest -sys.path.append('scripts/ivas_pytests/') -sys.path.append('scripts/ivas_pytests/tests/') -from il2mm import il2mm from cmp_custom import cmp_custom from cut_pcm import cut_samples from conftest import EncoderFrontend, DecoderFrontend @@ -67,7 +64,6 @@ agc_list = [0, 1] sample_rate_bw_idx_list = [('48', 'SWB'), ('48', 'WB'), ('32', 'WB')] AbsTol = '4' -ch_count_foa = 4 def check_and_makedir(dir_path): @@ -86,15 +82,16 @@ def check_and_makedir(dir_path): def test_bypass_enc( dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, - data_system_tests_path, + test_vector_path, reference_path, dut_base_path, ref_encoder_path, ref_decoder_path, update_ref, + keep_files, tag, fs, - bypass + bypass, ): if update_ref == 1 and bypass == 1: pytest.skip() @@ -108,9 +105,9 @@ def test_bypass_enc( output_config = "FOA" # enc - spar_foa_enc( + sba_enc( dut_encoder_frontend, - data_system_tests_path, + test_vector_path, ref_encoder_path, reference_path, dut_base_path, @@ -126,13 +123,12 @@ def test_bypass_enc( ) # dec - spar_foa_dec( + sba_dec( dut_decoder_frontend, ref_decoder_path, reference_path, dut_base_path, tag, - ch_count_foa, fs, ivas_br, dtx, @@ -140,7 +136,8 @@ def test_bypass_enc( bypass, agc, output_config, - update_ref + update_ref, + keep_files, ) @@ -150,20 +147,21 @@ def test_bypass_enc( @pytest.mark.parametrize("tag", tag_list) @pytest.mark.parametrize("fs", sample_rate_list) @pytest.mark.parametrize("agc", agc_list) -def test_spar_foa_enc_system( +def test_sba_enc_system( dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, - data_system_tests_path, + test_vector_path, reference_path, dut_base_path, ref_encoder_path, ref_decoder_path, update_ref, + keep_files, ivas_br, dtx, tag, fs, - agc + agc, ): tag = tag + fs + 'c' max_bw = "FB" @@ -176,9 +174,9 @@ def test_spar_foa_enc_system( cut_gain = "1.0" # enc - spar_foa_enc( + sba_enc( dut_encoder_frontend, - data_system_tests_path, + test_vector_path, ref_encoder_path, reference_path, dut_base_path, @@ -196,13 +194,12 @@ def test_spar_foa_enc_system( ) # dec - spar_foa_dec( + sba_dec( dut_decoder_frontend, ref_decoder_path, reference_path, dut_base_path, tag, - ch_count_foa, fs, ivas_br, dtx, @@ -210,7 +207,8 @@ def test_spar_foa_enc_system( bypass, agc, output_config, - update_ref + update_ref, + keep_files, ) @pytest.mark.create_ref @@ -219,14 +217,15 @@ def test_spar_foa_enc_system( def test_spar_hoa2_enc_system( dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, - data_system_tests_path, + test_vector_path, reference_path, dut_base_path, ref_encoder_path, ref_decoder_path, update_ref, + keep_files, ivas_br, - tag + tag, ): fs = '48' dtx = '0' @@ -239,9 +238,9 @@ def test_spar_hoa2_enc_system( output_config = "HOA2" # enc - spar_foa_enc( + sba_enc( dut_encoder_frontend, - data_system_tests_path, + test_vector_path, ref_encoder_path, reference_path, dut_base_path, @@ -258,13 +257,12 @@ def test_spar_hoa2_enc_system( ) # dec - spar_foa_dec( + sba_dec( dut_decoder_frontend, ref_decoder_path, reference_path, dut_base_path, tag, - ch_count_foa, fs, ivas_br, dtx, @@ -272,7 +270,8 @@ def test_spar_hoa2_enc_system( bypass, agc, output_config, - update_ref + update_ref, + keep_files, ) @pytest.mark.create_ref @@ -281,14 +280,15 @@ def test_spar_hoa2_enc_system( def test_spar_hoa3_enc_system( dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, - data_system_tests_path, + test_vector_path, reference_path, dut_base_path, ref_encoder_path, ref_decoder_path, update_ref, + keep_files, ivas_br, - tag + tag, ): fs = '48' dtx = '0' @@ -301,9 +301,9 @@ def test_spar_hoa3_enc_system( output_config = "HOA3" # enc - spar_foa_enc( + sba_enc( dut_encoder_frontend, - data_system_tests_path, + test_vector_path, ref_encoder_path, reference_path, dut_base_path, @@ -320,13 +320,12 @@ def test_spar_hoa3_enc_system( ) # dec - spar_foa_dec( + sba_dec( dut_decoder_frontend, ref_decoder_path, reference_path, dut_base_path, tag, - ch_count_foa, fs, ivas_br, dtx, @@ -334,7 +333,8 @@ def test_spar_hoa3_enc_system( bypass, agc, output_config, - update_ref + update_ref, + keep_files, ) @pytest.mark.create_ref @@ -342,19 +342,20 @@ def test_spar_hoa3_enc_system( @pytest.mark.parametrize("dtx", dtx_set) @pytest.mark.parametrize("tag", tag_list_bw_force) @pytest.mark.parametrize("sample_rate_bw_idx", sample_rate_bw_idx_list) -def test_spar_foa_enc_BWforce_system( +def test_sba_enc_BWforce_system( dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, - data_system_tests_path, + test_vector_path, reference_path, dut_base_path, ref_encoder_path, ref_decoder_path, update_ref, + keep_files, ivas_br, dtx, tag, - sample_rate_bw_idx + sample_rate_bw_idx, ): fs = sample_rate_bw_idx[0] bw = sample_rate_bw_idx[1] @@ -365,9 +366,9 @@ def test_spar_foa_enc_BWforce_system( output_config = "FOA" # enc - spar_foa_enc( + sba_enc( dut_encoder_frontend, - data_system_tests_path, + test_vector_path, ref_encoder_path, reference_path, dut_base_path, @@ -383,13 +384,12 @@ def test_spar_foa_enc_BWforce_system( ) # dec - spar_foa_dec( + sba_dec( dut_decoder_frontend, ref_decoder_path, reference_path, dut_base_path, tag, - ch_count_foa, fs, ivas_br, dtx, @@ -397,13 +397,14 @@ def test_spar_foa_enc_BWforce_system( bypass, agc, output_config, - update_ref + update_ref, + keep_files, ) ######################################################### ############ test function ############################## -def spar_foa_enc( +def sba_enc( encoder_frontend, test_vector_path, ref_encoder_path, @@ -424,8 +425,8 @@ def spar_foa_enc( ): ######### run cmd ##################################### - dut_out_dir = f"{dut_base_path}/spar_foa_bs/pkt" - ref_out_dir = f"{reference_path}/spar_foa_bs/pkt" + dut_out_dir = f"{dut_base_path}/sba_bs/pkt" + ref_out_dir = f"{reference_path}/sba_bs/pkt" check_and_makedir(dut_out_dir) check_and_makedir(ref_out_dir) @@ -512,13 +513,12 @@ def spar_foa_enc( ) -def spar_foa_dec( +def sba_dec( decoder_frontend, ref_decoder_path, reference_path, dut_base_path, tag, - ch_count, sampling_rate, ivas_br, dtx, @@ -527,7 +527,7 @@ def spar_foa_dec( agc, output_config, update_ref, - keep_files=False + keep_files, ): ######### run cmd ##################################### @@ -549,11 +549,14 @@ def spar_foa_dec( # to avoid conflicting names in case of parallel test execution, differentiate all cases long_tag_ext = f"_AGC{agc}_pca{bypass}" - dut_out_dir = f"{dut_base_path}/spar_foa_bs/raw/{tag_out}{long_tag_ext}" - ref_out_dir = f"{reference_path}/spar_foa_bs/raw/{tag_out}{short_tag_ext}" + dut_out_dir = f"{dut_base_path}/sba_bs/raw" + ref_out_dir = f"{reference_path}/sba_bs/raw" - dut_in_pkt = f"{dut_base_path}/spar_foa_bs/pkt/{tag_out}{long_tag_ext}.pkt" - ref_in_pkt = f"{reference_path}/spar_foa_bs/pkt/{tag_out}{short_tag_ext}.pkt" + dut_in_pkt = f"{dut_base_path}/sba_bs/pkt/{tag_out}{long_tag_ext}.pkt" + ref_in_pkt = f"{reference_path}/sba_bs/pkt/{tag_out}{short_tag_ext}.pkt" + + dut_out_raw = f"{dut_out_dir}/{tag_out}{long_tag_ext}.raw" + ref_out_raw = f"{ref_out_dir}/{tag_out}{short_tag_ext}.raw" check_and_makedir(dut_out_dir) check_and_makedir(ref_out_dir) @@ -566,48 +569,34 @@ def spar_foa_dec( output_config, sampling_rate, ref_in_pkt, - f"{ref_out_dir}/out.raw", + ref_out_raw, ) - # convert REF interleaved to multi-mono - il2mm(f"{ref_out_dir}/out.raw", ch_count, b_delete=not keep_files) - if update_ref == 0: # call DUT decoder decoder_frontend.run( output_config, sampling_rate, dut_in_pkt, - f"{dut_out_dir}/out.raw", + dut_out_raw, ) - il2mm(f"{dut_out_dir}/out.raw", ch_count, b_delete=not keep_files) - ######### compare cmd ##################################### end_skip_samples = '0' - test_fail = False - for count in range(ch_count): - ch_id = str(count + 1) - - # TEST - fsize1 = os.path.getsize(f"{dut_out_dir}/out{ch_id}ch.raw") - fsize2 = os.path.getsize(f"{ref_out_dir}/out{ch_id}ch.raw") - print(f"Want to compare {dut_out_dir}/out{ch_id}ch.raw ({fsize1} bytes) with {ref_out_dir}/out{ch_id}ch.raw ({fsize2} bytes)") - if cmp_custom( - f"{dut_out_dir}/out{ch_id}ch.raw", - f"{ref_out_dir}/out{ch_id}ch.raw", - "2", - AbsTol, - end_skip_samples - ) != 0: - test_fail = True - - ##File removal## + cmp_result, reason = cmp_custom( + dut_out_raw, + ref_out_raw, + "2", + AbsTol, + end_skip_samples + ) + + # report compare result + assert cmp_result == 0, reason + + # remove DUT output files when test result is OK (to save disk space) if not keep_files: os.remove(dut_in_pkt) - shutil.rmtree(dut_out_dir, ignore_errors=True) - - ##report failure - assert not test_fail + os.remove(dut_out_raw) diff --git a/tests/testconfig.py b/tests/testconfig.py new file mode 100644 index 0000000000000000000000000000000000000000..f4827a004f91b5e63a1330edc71c2da70fbd2967 --- /dev/null +++ b/tests/testconfig.py @@ -0,0 +1,37 @@ +__copyright__ = \ +""" +(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +__doc__ = \ +""" +To configure test modules. +""" + +PARAM_FILE = "scripts/config/self_test.prm"