From df740dabb0d7d8c3676b36b38ad7b6aebc4e1aa0 Mon Sep 17 00:00:00 2001 From: rtyag Date: Sat, 11 May 2024 17:42:20 +1000 Subject: [PATCH 1/8] addition of isar confirmance tests --- .gitlab-ci.yml | 28 ++++++++++++---- scripts/dec_isar_header.txt | 20 ++++++++++++ scripts/isar_post_rend_header.txt | 21 ++++++++++++ scripts/parse_commands.py | 48 +++++++++++++++++++++++++--- tests/conformance-test/conftest.py | 12 +++++++ tests/conformance-test/test_26252.py | 17 +++++++--- tests/renderer/utils.py | 35 ++++++++++++++++++++ tests/split_rendering/utils.py | 26 +++++++++------ 8 files changed, 181 insertions(+), 26 deletions(-) create mode 100644 scripts/dec_isar_header.txt create mode 100644 scripts/isar_post_rend_header.txt diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2dd073e563..4ba91eae6d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -513,7 +513,6 @@ build-codec-windows-msbuild: script: - *print-common-info-windows - *activate-WX-windows - - python .\scripts\strip_split_rendering.py - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug # --------------------------------------------------------------- @@ -1380,20 +1379,19 @@ ivas-conformance: - 123 script: - *print-common-info-windows - - python .\scripts\strip_split_rendering.py - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug - cp -force IVAS_cod.exe IVAS_cod_ref.exe - cp -force IVAS_dec.exe IVAS_dec_ref.exe - - cp -force IVAS_rend.exe IVAS_rend_ref.exe + - cp -force IVAS_rend.exe IVAS_rend_ref.exe # Reference creation - python tests/create_short_testvectors.py - python scripts/prepare_combined_format_inputs.py - - python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files - - python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --update_ref 1 -m create_ref_part2 --keep_files + - python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files + - python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py -v -n auto --update_ref 1 -m create_ref_part2 --keep_files # Output creation - - python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html + - python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html - python scripts/parse_commands.py report_cmd.html Readme_IVAS.txt # Copy input data and output ref data @@ -1402,10 +1400,13 @@ ivas-conformance: - if (Test-Path TMP_ENC) {rm -r -force TMP_ENC} - if (Test-Path TMP_JBM) {rm -r -force TMP_JBM} - if (Test-Path TMP_REND) {rm -r -force TMP_REND} + - if (Test-Path TMP_ISAR_POST_REND) {rm -r -force TMP_ISAR_POST_REND} + - if (Test-Path TMP_DEC_ISAR) {rm -r -force TMP_DEC_ISAR} - mkdir testvec - mkdir testvec/binauralRenderer_interface - mkdir testvec/testv - mkdir testvec/testv/renderer + - mkdir testvec/testv/split_renderer - mkdir testvec/bin - cp -force -ErrorAction Ignore scripts/testv/* testvec/testv - cp -r -force -ErrorAction Ignore scripts/ls_layouts testvec @@ -1415,6 +1416,7 @@ ivas-conformance: - cp -r -force -ErrorAction Ignore tests/ref testvec/testv/ref - cp -r -force -ErrorAction Ignore tests/dut/* testvec/testv/ref - cp -r -force -ErrorAction Ignore tests/renderer/cut testvec/testv/renderer/ref + - cp -r -force -ErrorAction Ignore tests/split_renderer/cut testvec/testv/split_renderer/ref # Remove redundant files - python scripts/cleanup_26252.py @@ -1425,9 +1427,12 @@ ivas-conformance: - cp Readme_IVAS_enc.txt testvec - cp Readme_IVAS_rend.txt testvec - cp Readme_IVAS_JBM_dec.txt testvec + - cp Readme_IVAS_ISAR_dec.txt testvec + - cp Readme_IVAS_ISAR_post_rend.txt testvec - cp IVAS_cod.exe testvec/bin - cp IVAS_dec.exe testvec/bin - cp IVAS_rend.exe testvec/bin + - cp ISAR_post_rend.exe testvec/bin # Test run generated scripts in testvec @@ -1447,6 +1452,8 @@ ivas-conformance: - Readme_IVAS_enc.txt - Readme_IVAS_rend.txt - Readme_IVAS_JBM_dec.txt + - Readme_IVAS_ISAR_dec.txt + - Readme_IVAS_ISAR_post_rend.txt expose_as: "Draft IVAS conformance" reports: junit: report-junit.xml @@ -1471,7 +1478,7 @@ ivas-conformance-linux: # Reference creation - python3 tests/create_short_testvectors.py - python3 scripts/prepare_combined_format_inputs.py - - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py" + - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py" - python3 -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files - python3 -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref_part2 --keep_files @@ -1485,6 +1492,7 @@ ivas-conformance-linux: - mkdir testvec/binauralRenderer_interface - mkdir testvec/testv - mkdir testvec/testv/renderer + - mkdir testvec/testv/split_renderer - mkdir testvec/bin - cp -r scripts/testv/* testvec/testv - cp -r scripts/ls_layouts testvec @@ -1494,6 +1502,7 @@ ivas-conformance-linux: - cp -r tests/ref testvec/testv/ref - cp -r tests/dut/* testvec/testv/ref - cp -r tests/renderer/cut testvec/testv/renderer/ref + - cp -r tests/split_renderer/cut testvec/testv/split_renderer/ref # Remove redundant files - python3 scripts/cleanup_26252.py @@ -1504,6 +1513,8 @@ ivas-conformance-linux: - cp Readme_IVAS_enc.txt testvec - cp Readme_IVAS_rend.txt testvec - cp Readme_IVAS_JBM_dec.txt testvec + - cp Readme_IVAS_ISAR_dec.txt testvec + - cp Readme_IVAS_ISAR_post_rend.txt testvec # Create GCOV execs for coverage analysis - make clean @@ -1512,6 +1523,7 @@ ivas-conformance-linux: - cp IVAS_cod testvec/bin - cp IVAS_dec testvec/bin - cp IVAS_rend testvec/bin + - cp ISAR_post_rend testvec/bin # Test run generated scripts in testvec - cd testvec @@ -1540,6 +1552,8 @@ ivas-conformance-linux: - Readme_IVAS_enc.txt - Readme_IVAS_rend.txt - Readme_IVAS_JBM_dec.txt + - Readme_IVAS_ISAR_dec.txt + - Readme_IVAS_ISAR_post_rend.txt - coverage.info - coverage expose_as: "Draft IVAS conformance -- Linux" diff --git a/scripts/dec_isar_header.txt b/scripts/dec_isar_header.txt new file mode 100644 index 0000000000..5489702bcf --- /dev/null +++ b/scripts/dec_isar_header.txt @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit -1 +fi + +CUT_DEC_BIN=$1 +DIFF_BIN="diff" + +TESTV_PATH="." +REF_PATH="./testv" +CUT_PATH="./TMP_DEC_ISAR" +LOG_FILE=Readme_IVAS_ISAR_dec_log.txt + +rm -rf tmp +rm -rf $CUT_PATH +mkdir -p $CUT_PATH +mkdir -p $CUT_PATH/dut/isar/pre_rend_out + diff --git a/scripts/isar_post_rend_header.txt b/scripts/isar_post_rend_header.txt new file mode 100644 index 0000000000..9b74645a84 --- /dev/null +++ b/scripts/isar_post_rend_header.txt @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit -1 +fi + +CUT_ISAR_POST_REND_BIN=$1 +DIFF_BIN="diff" + +TESTV_PATH="." +REF_PATH="./testv" +CUT_PATH="./TMP_ISAR_POST_REND" +LOG_FILE=Readme_IVAS_isar_post_rend_log.txt + +rm -rf tmp +rm -rf $CUT_PATH +mkdir -p $CUT_PATH/split_renderer/cut +mkdir -p $CUT_PATH/split_renderer/data + + diff --git a/scripts/parse_commands.py b/scripts/parse_commands.py index 6d10698cfc..ce2b3c1b53 100644 --- a/scripts/parse_commands.py +++ b/scripts/parse_commands.py @@ -24,6 +24,7 @@ if __name__ == '__main__': cmds_enc=[] cmds_dec=[] cmds_rend=[] + cmds_isar_post_rend=[] if path.isdir(input): @@ -37,9 +38,10 @@ if __name__ == '__main__': cmds_enc.extend(re.findall(r"DUT encoder command:\\n\\t(.*?)\\n", line)) cmds_dec.extend(re.findall(r"DUT decoder command:\\n\\t(.*?)\\n", line)) cmds_rend.extend(re.findall(r"Running command\\n(.*?)\\n", line)) + cmds_isar_post_rend.extend(re.findall(r"Running ISAR post renderer command\\n(.*?)\\n", line)) # If pytest-html < v4 is used, the parsing will fail and render empty lists. This is a work-around in case that happens. - if all(not x for x in [cmds_enc, cmds_dec, cmds_rend]): + if all(not x for x in [cmds_enc, cmds_dec, cmds_rend, cmds_isar_post_rend]): for html_report in input: with open(html_report,'r') as infile: enc_cmd = False @@ -54,19 +56,26 @@ if __name__ == '__main__': dec_cmd = False elif rend_cmd: cmds_rend.append(line) - rend_cmd = False + rend_cmd = False + elif cmds_isar_post_rend: + cmds_isar_post_rend.append(line) + cmds_isar_post_rend = False else: if "DUT encoder command" in line: enc_cmd = True elif "DUT decoder command" in line: dec_cmd = True elif "Running command" in line: - rend_cmd = True + rend_cmd = True + elif "Running ISAR post renderer command" in line: + cmds_isar_post_rend = True # Sort lists to keep deterministic order between runs cmds_enc.sort() cmds_dec.sort() cmds_rend.sort() + cmds_ivas_isar_dec.sort() + cmds_isar_post_rend.sort() with open(txt_file.replace('.','_enc.'),'w', newline='\n') as outfile: with open('scripts/enc_header.txt','r') as header: @@ -90,11 +99,13 @@ if __name__ == '__main__': with open('scripts/script_footer.txt','r') as footer: outfile.write(footer.read()) - with open(txt_file.replace('.','_dec.'),'w', newline='\n') as outfile_dec, open(txt_file.replace('.','_JBM_dec.'),'w', newline='\n') as outfile_jbm: + with open(txt_file.replace('.','_dec.'),'w', newline='\n') as outfile_dec, open(txt_file.replace('.','_JBM_dec.'),'w', newline='\n') as outfile_jbm, open(txt_file.replace('.','_ISAR_dec.'),'w', newline='\n') as outfile_isar: with open('scripts/dec_header.txt','r') as header: outfile_dec.write(header.read()) with open('scripts/jbm_header.txt','r') as header: outfile_jbm.write(header.read()) + with open('scripts/dec_isar_header.txt','r') as header: + outfile_isar.write(header.read()) for cmd in cmds_dec: absolute_out = re.search(r"\s(([\S]+)(.wav))$", cmd) @@ -115,6 +126,8 @@ if __name__ == '__main__': if 'VOIP' in cmd: outfile = outfile_jbm + elif 'BINAURAL_SPLIT' in cmd: + outfile = outfile_isar else: outfile = outfile_dec @@ -156,4 +169,29 @@ if __name__ == '__main__': outfile.write('$DIFF_BIN '+out.group(1).replace(CUT_PATH + r'/renderer/cut',REF_PATH + r'/renderer/ref')+' '+out.group(1)+' >> $LOG_FILE 2>&1\n') outfile.write('\n') with open('scripts/script_footer.txt','r') as footer: - outfile.write(footer.read()) \ No newline at end of file + outfile.write(footer.read()) + + with open(txt_file.replace('.','_ISAR_post_rend.'),'w', newline='\n') as outfile: + with open('scripts/isar_post_rend_header.txt','r') as header: + outfile.write(header.read()) + for cmd in cmds_isar_post_rend: + args = [] + for arg in cmd.split(): + # Adjust file arguments, pass other arguments as they are + if path.exists(arg): + arg = path.relpath(arg).replace('\\','/') + arg = re.sub('ISAR_post_rend(.exe)?', '$CUT_ISAR_POST_REND_BIN', arg) + arg = re.sub('scripts', TESTV_PATH, arg) + arg = re.sub('tests/split_renderer/data', TESTV_PATH + r'split_renderer/data/', arg) + arg = re.sub('tests', CUT_PATH, arg) + args.append(arg) + cmd = ' '.join(args) + + if "cut" in cmd: + outfile.write(cmd+'\n') + out = re.search(r"-o\s(([\S]+)(.wav|.raw|.pcm))", cmd) + if out and "cut" in out.group(1): + outfile.write('$DIFF_BIN '+out.group(1).replace(CUT_PATH + r'/split_renderer/cut',REF_PATH + r'/split_renderer/ref')+' '+out.group(1)+' >> $LOG_FILE 2>&1\n') + outfile.write('\n') + with open('scripts/script_footer.txt','r') as footer: + outfile.write(footer.read()) \ No newline at end of file diff --git a/tests/conformance-test/conftest.py b/tests/conformance-test/conftest.py index 5eea228c45..1550e6c3fb 100644 --- a/tests/conformance-test/conftest.py +++ b/tests/conformance-test/conftest.py @@ -54,6 +54,13 @@ def pytest_addoption(parser): type=pathlib.Path, default='./bin/IVAS_rend', ) + parser.addoption( + "--isar_post_renderer_path", + action="store", + help="path to renderer binary ISAR_post_rend(.exe)", + type=pathlib.Path, + default='./bin/ISAR_post_rend', + ) @pytest.fixture(scope="session") @@ -69,3 +76,8 @@ def decoder_path(request) -> str: @pytest.fixture(scope="session") def renderer_path(request) -> str: return str(request.config.option.renderer_path.absolute()) + + +@pytest.fixture(scope="session") +def isar_post_renderer_path(request) -> str: + return str(request.config.option.isar_post_renderer_path.absolute()) diff --git a/tests/conformance-test/test_26252.py b/tests/conformance-test/test_26252.py index ac5ca6d799..e3b1371d57 100644 --- a/tests/conformance-test/test_26252.py +++ b/tests/conformance-test/test_26252.py @@ -59,7 +59,7 @@ def replace_paths(instr, testv_path, ref_path, cut_path): test_dict = {} TEST_DIR = "." -scripts=["Readme_IVAS_enc.txt", "Readme_IVAS_dec.txt", "Readme_IVAS_rend.txt", "Readme_IVAS_JBM_dec.txt"] +scripts=["Readme_IVAS_enc.txt", "Readme_IVAS_dec.txt", "Readme_IVAS_rend.txt", "Readme_IVAS_JBM_dec.txt", "Readme_IVAS_ISAR_dec.txt", "Readme_IVAS_isar_post_rend.txt"] for s in scripts: with open(os.path.join(TEST_DIR, s), "r", encoding="UTF-8") as fp: @@ -68,6 +68,7 @@ for s in scripts: dec_opts = "" diff_opts = "" rend_opts = "" + isar_post_rend_opts = "" testv_path = "" ref_path = "" cut_path = "" @@ -86,18 +87,21 @@ for s in scripts: if line.startswith("$CUT_DEC_BIN"): dec_opts = line if line.startswith("$CUT_REND_BIN"): - rend_opts = line + rend_opts = line + if line.startswith("$CUT_ISAR_POST_REND_BIN"): + isar_post_rend_opts = line if line.startswith("$DIFF_BIN"): diff_opts = line tag = s + "--" + diff_opts.split()[2].split('/')[-1] if tag in test_dict: print("non-unique tag found - ignoring new entry") continue - test_dict[tag] = (enc_opts, dec_opts, rend_opts, diff_opts, testv_path, ref_path, cut_path) + test_dict[tag] = (enc_opts, dec_opts, rend_opts, isar_post_rend_opts, diff_opts, testv_path, ref_path, cut_path) tag = "" enc_opts = "" dec_opts = "" rend_opts = "" + isar_post_rend_opts = "" diff_opts = "" for proc in preproc: proc = replace_paths(proc, testv_path, ref_path, cut_path) @@ -106,9 +110,9 @@ for s in scripts: Path(path_arg).mkdir(parents=True, exist_ok=True) @pytest.mark.parametrize("test_tag", list(test_dict.keys())) -def test_26252(test_tag, encoder_path, decoder_path, renderer_path): +def test_26252(test_tag, encoder_path, decoder_path, renderer_path, isar_post_renderer_path): - enc_opts, dec_opts, rend_opts, diff_opts, testv_path, ref_path, cut_path = test_dict[test_tag] + enc_opts, dec_opts, rend_opts, isar_post_rend_opts, diff_opts, testv_path, ref_path, cut_path = test_dict[test_tag] if enc_opts: enc_opts = replace_paths(enc_opts, testv_path, ref_path, cut_path) @@ -119,6 +123,9 @@ def test_26252(test_tag, encoder_path, decoder_path, renderer_path): if rend_opts: rend_opts = replace_paths(rend_opts, testv_path, ref_path, cut_path) subprocess.run([renderer_path] + rend_opts.split()[1:], check = True) + if isar_post_rend_opts: + rend_opts = replace_paths(isar_post_rend_opts, testv_path, ref_path, cut_path) + subprocess.run([isar_post_renderer_path] + isar_post_rend_opts.split()[1:], check = True) diff_opts = replace_paths(diff_opts, testv_path, ref_path, cut_path) result = True diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index a538df7ee8..9f2f9050cc 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -62,6 +62,41 @@ def run_cmd(cmd, env=None): f"Command returned non-zero exit status ({e.returncode}): {' '.join(e.cmd)}\n{e.stderr}\n{e.stdout}" ) +def run_isar_ext_rend_cmd(cmd, env=None): + logging.info(f"\nRunning ISAR EXT REND command\n{' '.join(cmd)}\n") + try: + sp.run(cmd, check=True, capture_output=True, text=True, env=env) + except sp.CalledProcessError as e: + raise SystemError( + f"Command returned non-zero exit status ({e.returncode}): {' '.join(e.cmd)}\n{e.stderr}\n{e.stdout}" + ) + +def run_ivas_isar_enc_cmd(cmd, env=None): + logging.info(f"\nRunning IVAS ISAR encoder command\n{' '.join(cmd)}\n") + try: + sp.run(cmd, check=True, capture_output=True, text=True, env=env) + except sp.CalledProcessError as e: + raise SystemError( + f"Command returned non-zero exit status ({e.returncode}): {' '.join(e.cmd)}\n{e.stderr}\n{e.stdout}" + ) + +def run_ivas_isar_dec_cmd(cmd, env=None): + logging.info(f"\nDUT decoder command\n{' '.join(cmd)}\n") + try: + sp.run(cmd, check=True, capture_output=True, text=True, env=env) + except sp.CalledProcessError as e: + raise SystemError( + f"Command returned non-zero exit status ({e.returncode}): {' '.join(e.cmd)}\n{e.stderr}\n{e.stdout}" + ) + +def run_isar_post_rend_cmd(cmd, env=None): + logging.info(f"\nRunning ISAR post renderer command\n{' '.join(cmd)}\n") + try: + sp.run(cmd, check=True, capture_output=True, text=True, env=env) + except sp.CalledProcessError as e: + raise SystemError( + f"Command returned non-zero exit status ({e.returncode}): {' '.join(e.cmd)}\n{e.stderr}\n{e.stdout}" + ) def check_BE( test_info, diff --git a/tests/split_rendering/utils.py b/tests/split_rendering/utils.py index 270746b0d5..424e39e3a3 100644 --- a/tests/split_rendering/utils.py +++ b/tests/split_rendering/utils.py @@ -39,7 +39,7 @@ from typing import Tuple import numpy as np import pytest -from tests.renderer.utils import check_BE, run_cmd, test_info +from tests.renderer.utils import check_BE, run_cmd, test_info, run_ivas_isar_enc_cmd, run_ivas_isar_dec_cmd, run_isar_post_rend_cmd, run_isar_ext_rend_cmd from tests.split_rendering.constants import * sys.path.append(SCRIPTS_DIR) @@ -172,10 +172,14 @@ def run_full_chain_split_rendering( with TemporaryDirectory() as tmp_dir: tmp_dir = Path(tmp_dir) cut_in_file = tmp_dir.joinpath("cut_input.wav") - ivas_bitstream = tmp_dir.joinpath("ivas.192") - split_bitstream = tmp_dir.joinpath("split.bit") + #ivas_bitstream = tmp_dir.joinpath("ivas.192") + ivas_bitstream_stem = f"{in_fmt}_{bitrate}bps_ivas.192" + #split_bitstream = tmp_dir.joinpath("split.bit") + split_bitstream_stem = f"{in_fmt}_{bitrate}bps_{renderer_fmt}_{pre_trajectory.stem}_split_full_{post_trajectory.stem}_config_{render_config.stem}_prerfr_{pre_rend_fr}_postrfr_{post_rend_fr}_split.bit" if renderer_fmt == "BINAURAL_SPLIT_PCM": - split_md_file = tmp_dir.joinpath("split_md.bin") + #split_md_file = tmp_dir.joinpath("split_md.bin") + split_md_file_stem = f"{in_fmt}_{bitrate}bps_{renderer_fmt}_{pre_trajectory.stem}_split_full_{post_trajectory.stem}_config_{render_config.stem}_prerfr_{pre_rend_fr}_postrfr_{post_rend_fr}_split_md.bit" + out_file_stem = f"{in_fmt}_{bitrate}bps_{renderer_fmt}_{pre_trajectory.stem}_split_full_{post_trajectory.stem}_config_{render_config.stem}_prerfr_{pre_rend_fr}_postrfr_{post_rend_fr}_.wav" if test_info.config.option.create_ref: @@ -183,7 +187,11 @@ def run_full_chain_split_rendering( else: output_path_base = OUTPUT_PATH_CUT + ivas_bitstream = output_path_base.joinpath(ivas_bitstream_stem) + split_bitstream = output_path_base.joinpath(split_bitstream_stem) out_file = output_path_base.joinpath(out_file_stem) + if renderer_fmt == "BINAURAL_SPLIT_PCM": + split_md_file = output_path_base.joinpath(split_md_file_stem) # check for metadata files if in_fmt.upper().startswith("OSBA"): @@ -216,7 +224,7 @@ def run_full_chain_split_rendering( cmd[1:1] = FORMAT_TO_IVAS_COD_FORMAT[in_fmt] - run_cmd(cmd) + run_ivas_isar_enc_cmd(cmd) # decode to split-rendering bitstream cmd = SPLIT_PRE_DEC_CMD[:] @@ -234,7 +242,7 @@ def run_full_chain_split_rendering( if renderer_fmt == "BINAURAL_SPLIT_PCM": cmd[5:5] = ["-om", str(split_md_file)] - run_cmd(cmd) + run_ivas_isar_dec_cmd(cmd) # run split renderer cmd = SPLIT_POST_REND_CMD[:] @@ -251,7 +259,7 @@ def run_full_chain_split_rendering( if renderer_fmt == "BINAURAL_SPLIT_PCM": cmd[7:7] = ["-im", str(split_md_file)] - run_cmd(cmd) + run_isar_post_rend_cmd(cmd) if test_info.config.option.create_cut: # CUT creation mode will run a comparison with REF @@ -343,7 +351,7 @@ def run_external_split_rendering( if in_meta_files: cmd[9:9] = ["-im", *in_meta_files] - run_cmd(cmd) + run_isar_ext_rend_cmd(cmd) # run split renderer cmd = SPLIT_POST_REND_CMD[:] @@ -363,7 +371,7 @@ def run_external_split_rendering( if plc_error_pattern: cmd[1:1] = ["-prbfi", str(plc_error_pattern)] - run_cmd(cmd) + run_isar_ext_rend_cmd(cmd) if test_info.config.option.create_cut: # CUT creation mode will run a comparison with REF -- GitLab From a8e0807f9ded7d759f9b1dd09effcc7a23357b6f Mon Sep 17 00:00:00 2001 From: rtyag Date: Sat, 11 May 2024 17:50:08 +1000 Subject: [PATCH 2/8] have unique name for decoder input --- tests/split_rendering/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/split_rendering/utils.py b/tests/split_rendering/utils.py index 424e39e3a3..d1d95a6242 100644 --- a/tests/split_rendering/utils.py +++ b/tests/split_rendering/utils.py @@ -173,7 +173,7 @@ def run_full_chain_split_rendering( tmp_dir = Path(tmp_dir) cut_in_file = tmp_dir.joinpath("cut_input.wav") #ivas_bitstream = tmp_dir.joinpath("ivas.192") - ivas_bitstream_stem = f"{in_fmt}_{bitrate}bps_ivas.192" + ivas_bitstream_stem = f"{in_fmt}_{bitrate}bps_{renderer_fmt}_{pre_trajectory.stem}_split_full_{post_trajectory.stem}_config_{render_config.stem}_prerfr_{pre_rend_fr}_postrfr_{post_rend_fr}_ivas.192" #split_bitstream = tmp_dir.joinpath("split.bit") split_bitstream_stem = f"{in_fmt}_{bitrate}bps_{renderer_fmt}_{pre_trajectory.stem}_split_full_{post_trajectory.stem}_config_{render_config.stem}_prerfr_{pre_rend_fr}_postrfr_{post_rend_fr}_split.bit" if renderer_fmt == "BINAURAL_SPLIT_PCM": -- GitLab From dc307c3e3c9086989903b443f98739cf9076b846 Mon Sep 17 00:00:00 2001 From: rtyag Date: Sun, 12 May 2024 23:36:56 +1000 Subject: [PATCH 3/8] fixes to ISAR conformance tests --- .gitlab-ci.yml | 22 +++++++++------ scripts/cleanup_26252.py | 2 +- scripts/dec_isar_header.txt | 3 ++- scripts/isar_post_rend_header.txt | 4 +-- scripts/parse_commands.py | 27 ++++++++++++++----- tests/conformance-test/test_26252.py | 4 +-- tests/renderer/utils.py | 2 +- tests/split_rendering/constants.py | 8 ++++++ tests/split_rendering/test_split_rendering.py | 8 ++++++ tests/split_rendering/utils.py | 8 +++--- 10 files changed, 63 insertions(+), 25 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4ba91eae6d..b3359459eb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1387,8 +1387,9 @@ ivas-conformance: # Reference creation - python tests/create_short_testvectors.py - python scripts/prepare_combined_format_inputs.py - - python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files - - python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py -v -n auto --update_ref 1 -m create_ref_part2 --keep_files + - python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files + - python -m pytest tests/split_rendering/test_split_rendering.py -v -n auto --create_ref --keep_files + - python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --update_ref 1 -m create_ref_part2 --keep_files # Output creation - python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html @@ -1406,7 +1407,7 @@ ivas-conformance: - mkdir testvec/binauralRenderer_interface - mkdir testvec/testv - mkdir testvec/testv/renderer - - mkdir testvec/testv/split_renderer + - mkdir testvec/testv/split_rendering - mkdir testvec/bin - cp -force -ErrorAction Ignore scripts/testv/* testvec/testv - cp -r -force -ErrorAction Ignore scripts/ls_layouts testvec @@ -1416,7 +1417,9 @@ ivas-conformance: - cp -r -force -ErrorAction Ignore tests/ref testvec/testv/ref - cp -r -force -ErrorAction Ignore tests/dut/* testvec/testv/ref - cp -r -force -ErrorAction Ignore tests/renderer/cut testvec/testv/renderer/ref - - cp -r -force -ErrorAction Ignore tests/split_renderer/cut testvec/testv/split_renderer/ref + - cp -r -force -ErrorAction Ignore tests/split_rendering/cut testvec/testv/split_rendering/ref + - cp -r -force -ErrorAction Ignore tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs + - cp -r -force -ErrorAction Ignore tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns # Remove redundant files - python scripts/cleanup_26252.py @@ -1478,12 +1481,13 @@ ivas-conformance-linux: # Reference creation - python3 tests/create_short_testvectors.py - python3 scripts/prepare_combined_format_inputs.py - - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py" + - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py" - python3 -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files + - python3 -m pytest tests/split_rendering/test_split_rendering.py -v -n auto --create_ref --keep_files - python3 -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref_part2 --keep_files # Output creation - - python3 -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html + - python3 -m pytest $TEST_SET tests/split_rendering/test_split_rendering.py -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html - python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt # Copy input data and output ref data @@ -1492,7 +1496,7 @@ ivas-conformance-linux: - mkdir testvec/binauralRenderer_interface - mkdir testvec/testv - mkdir testvec/testv/renderer - - mkdir testvec/testv/split_renderer + - mkdir testvec/testv/split_rendering - mkdir testvec/bin - cp -r scripts/testv/* testvec/testv - cp -r scripts/ls_layouts testvec @@ -1502,7 +1506,9 @@ ivas-conformance-linux: - cp -r tests/ref testvec/testv/ref - cp -r tests/dut/* testvec/testv/ref - cp -r tests/renderer/cut testvec/testv/renderer/ref - - cp -r tests/split_renderer/cut testvec/testv/split_renderer/ref + - cp -r tests/split_rendering/cut testvec/testv/split_rendering/ref + - cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs + - cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns # Remove redundant files - python3 scripts/cleanup_26252.py diff --git a/scripts/cleanup_26252.py b/scripts/cleanup_26252.py index df6c88ae98..2df321a1d4 100644 --- a/scripts/cleanup_26252.py +++ b/scripts/cleanup_26252.py @@ -4,7 +4,7 @@ from pathlib import Path import os import glob -files = ['Readme_IVAS_JBM_dec.txt','Readme_IVAS_dec.txt','Readme_IVAS_enc.txt','Readme_IVAS_rend.txt']; +files = ['Readme_IVAS_JBM_dec.txt','Readme_IVAS_dec.txt','Readme_IVAS_enc.txt','Readme_IVAS_rend.txt', 'Readme_IVAS_ISAR_dec.txt', 'Readme_IVAS_ISAR_post_rend.txt']; used_files = [] kept = 0 diff --git a/scripts/dec_isar_header.txt b/scripts/dec_isar_header.txt index 5489702bcf..454383382f 100644 --- a/scripts/dec_isar_header.txt +++ b/scripts/dec_isar_header.txt @@ -16,5 +16,6 @@ LOG_FILE=Readme_IVAS_ISAR_dec_log.txt rm -rf tmp rm -rf $CUT_PATH mkdir -p $CUT_PATH -mkdir -p $CUT_PATH/dut/isar/pre_rend_out +mkdir -p $CUT_PATH/split_rendering/cut + diff --git a/scripts/isar_post_rend_header.txt b/scripts/isar_post_rend_header.txt index 9b74645a84..ed7eb20988 100644 --- a/scripts/isar_post_rend_header.txt +++ b/scripts/isar_post_rend_header.txt @@ -15,7 +15,7 @@ LOG_FILE=Readme_IVAS_isar_post_rend_log.txt rm -rf tmp rm -rf $CUT_PATH -mkdir -p $CUT_PATH/split_renderer/cut -mkdir -p $CUT_PATH/split_renderer/data +mkdir -p $CUT_PATH/split_rendering/cut + diff --git a/scripts/parse_commands.py b/scripts/parse_commands.py index ce2b3c1b53..8b630337b5 100644 --- a/scripts/parse_commands.py +++ b/scripts/parse_commands.py @@ -47,6 +47,7 @@ if __name__ == '__main__': enc_cmd = False dec_cmd = False rend_cmd = False + isar_post_rend_cmd = False for line in infile.readlines(): if enc_cmd: cmds_enc.append(line) @@ -57,9 +58,9 @@ if __name__ == '__main__': elif rend_cmd: cmds_rend.append(line) rend_cmd = False - elif cmds_isar_post_rend: + elif isar_post_rend_cmd: cmds_isar_post_rend.append(line) - cmds_isar_post_rend = False + isar_post_rend_cmd = False else: if "DUT encoder command" in line: enc_cmd = True @@ -68,13 +69,12 @@ if __name__ == '__main__': elif "Running command" in line: rend_cmd = True elif "Running ISAR post renderer command" in line: - cmds_isar_post_rend = True + isar_post_rend_cmd = True # Sort lists to keep deterministic order between runs cmds_enc.sort() cmds_dec.sort() cmds_rend.sort() - cmds_ivas_isar_dec.sort() cmds_isar_post_rend.sort() with open(txt_file.replace('.','_enc.'),'w', newline='\n') as outfile: @@ -117,10 +117,15 @@ if __name__ == '__main__': arg = re.sub('IVAS_dec(.exe)?', '$CUT_DEC_BIN', arg) arg = re.sub('scripts', TESTV_PATH, arg) arg = re.sub('tests/ref', REF_PATH + r'/ref', arg) # For .fer cases the bitstream is in ref + arg = re.sub('tests/split_rendering/renderer_configs', REF_PATH + r'/split_rendering/renderer_configs', arg) + if re.search("^tests.*192$",arg): + arg = re.sub('tests/split_rendering/cut', REF_PATH + r'/split_rendering/ref', arg) if re.search("\.wav$",arg): arg = re.sub('tests', CUT_PATH, arg) # Output argument for decoder ends with .wav else: arg = re.sub('tests/dut', REF_PATH + r'/ref', arg) # Input argument + if re.search("^tests.*bit$",arg): + arg = re.sub('tests', CUT_PATH, arg) # Output argument for decoder ends with .wav args.append(arg) cmd = ' '.join(args) @@ -133,6 +138,8 @@ if __name__ == '__main__': outfile.write(cmd+'\n') out = re.search(r"\s(([\S]+)(.wav))$", cmd) + isar_out = re.search(r"\s(([\S]+)(.bit))$", cmd) + isar_md_out = re.search(r"-om\s(([\S]+)(.bit))", cmd) if out: diff_cmds=[] for output in glob.glob(absolute_out.group(1) + '*'): @@ -140,11 +147,18 @@ if __name__ == '__main__': output = re.sub('tests', CUT_PATH, output) diff_cmds.append('$DIFF_BIN '+output.replace(CUT_PATH + r'/dut',REF_PATH + r'/ref')+' '+output+' >> $LOG_FILE 2>&1') outfile.write(('; ').join(diff_cmds)) + if isar_out and "cut" in isar_out.group(1): + outfile.write('$DIFF_BIN '+isar_out.group(1).replace(CUT_PATH + r'/split_rendering/cut',REF_PATH + r'/split_rendering/ref')+' '+isar_out.group(1)+' >> $LOG_FILE 2>&1\n') + if isar_md_out and "cut" in isar_md_out.group(1): + outfile.write('$DIFF_BIN '+isar_md_out.group(1).replace(CUT_PATH + r'/split_rendering/cut',REF_PATH + r'/split_rendering/ref')+' '+isar_md_out.group(1)+' >> $LOG_FILE 2>&1\n') + outfile.write('\n\n') with open('scripts/script_footer.txt','r') as footer: outfile_dec.write(footer.read()) footer.seek(0) outfile_jbm.write(footer.read()) + footer.seek(0) + outfile_isar.write(footer.read()) with open(txt_file.replace('.','_rend.'),'w', newline='\n') as outfile: with open('scripts/rend_header.txt','r') as header: @@ -182,7 +196,8 @@ if __name__ == '__main__': arg = path.relpath(arg).replace('\\','/') arg = re.sub('ISAR_post_rend(.exe)?', '$CUT_ISAR_POST_REND_BIN', arg) arg = re.sub('scripts', TESTV_PATH, arg) - arg = re.sub('tests/split_renderer/data', TESTV_PATH + r'split_renderer/data/', arg) + if re.search("^tests.*bit$",arg): + arg = re.sub('tests/split_rendering/cut', REF_PATH + r'/split_rendering/ref', arg) arg = re.sub('tests', CUT_PATH, arg) args.append(arg) cmd = ' '.join(args) @@ -191,7 +206,7 @@ if __name__ == '__main__': outfile.write(cmd+'\n') out = re.search(r"-o\s(([\S]+)(.wav|.raw|.pcm))", cmd) if out and "cut" in out.group(1): - outfile.write('$DIFF_BIN '+out.group(1).replace(CUT_PATH + r'/split_renderer/cut',REF_PATH + r'/split_renderer/ref')+' '+out.group(1)+' >> $LOG_FILE 2>&1\n') + outfile.write('$DIFF_BIN '+out.group(1).replace(CUT_PATH + r'/split_rendering/cut',REF_PATH + r'/split_rendering/ref')+' '+out.group(1)+' >> $LOG_FILE 2>&1\n') outfile.write('\n') with open('scripts/script_footer.txt','r') as footer: outfile.write(footer.read()) \ No newline at end of file diff --git a/tests/conformance-test/test_26252.py b/tests/conformance-test/test_26252.py index e3b1371d57..872c634b39 100644 --- a/tests/conformance-test/test_26252.py +++ b/tests/conformance-test/test_26252.py @@ -59,7 +59,7 @@ def replace_paths(instr, testv_path, ref_path, cut_path): test_dict = {} TEST_DIR = "." -scripts=["Readme_IVAS_enc.txt", "Readme_IVAS_dec.txt", "Readme_IVAS_rend.txt", "Readme_IVAS_JBM_dec.txt", "Readme_IVAS_ISAR_dec.txt", "Readme_IVAS_isar_post_rend.txt"] +scripts=["Readme_IVAS_enc.txt", "Readme_IVAS_dec.txt", "Readme_IVAS_rend.txt", "Readme_IVAS_JBM_dec.txt", "Readme_IVAS_ISAR_dec.txt", "Readme_IVAS_ISAR_post_rend.txt"] for s in scripts: with open(os.path.join(TEST_DIR, s), "r", encoding="UTF-8") as fp: @@ -124,7 +124,7 @@ def test_26252(test_tag, encoder_path, decoder_path, renderer_path, isar_post_re rend_opts = replace_paths(rend_opts, testv_path, ref_path, cut_path) subprocess.run([renderer_path] + rend_opts.split()[1:], check = True) if isar_post_rend_opts: - rend_opts = replace_paths(isar_post_rend_opts, testv_path, ref_path, cut_path) + isar_post_rend_opts = replace_paths(isar_post_rend_opts, testv_path, ref_path, cut_path) subprocess.run([isar_post_renderer_path] + isar_post_rend_opts.split()[1:], check = True) diff_opts = replace_paths(diff_opts, testv_path, ref_path, cut_path) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 9f2f9050cc..71cd852d59 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -81,7 +81,7 @@ def run_ivas_isar_enc_cmd(cmd, env=None): ) def run_ivas_isar_dec_cmd(cmd, env=None): - logging.info(f"\nDUT decoder command\n{' '.join(cmd)}\n") + logging.info(f"\nDUT decoder command:\n\t{' '.join(cmd)}\n") try: sp.run(cmd, check=True, capture_output=True, text=True, env=env) except sp.CalledProcessError as e: diff --git a/tests/split_rendering/constants.py b/tests/split_rendering/constants.py index 11c76e3d62..da06eb6ceb 100644 --- a/tests/split_rendering/constants.py +++ b/tests/split_rendering/constants.py @@ -31,6 +31,7 @@ """ from pathlib import Path +import platform from tests.renderer.constants import ( BIN_SUFFIX_MERGETARGET, @@ -41,6 +42,13 @@ from tests.renderer.constants import ( METADATA_SCENES_TO_TEST, ) +if platform.system() == "Windows": + EXE_SUFFIX = ".exe" +elif platform.system() in ["Linux", "Darwin"]: + EXE_SUFFIX = "" +else: + assert False, f"Unsupported platform {platform.system()}" + """ Set up paths """ TESTS_DIR = Path(__file__).parent RENDER_CFG_DIR = TESTS_DIR.joinpath("renderer_configs").resolve() diff --git a/tests/split_rendering/test_split_rendering.py b/tests/split_rendering/test_split_rendering.py index b5886e478f..3bf9db3b1c 100644 --- a/tests/split_rendering/test_split_rendering.py +++ b/tests/split_rendering/test_split_rendering.py @@ -53,6 +53,7 @@ def test_ambisonics_full_chain_split( in_fmt=in_fmt, bitrate=bitrate, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), + binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, ) @@ -92,6 +93,7 @@ def test_multichannel_full_chain_split( in_fmt=in_fmt, bitrate=bitrate, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), + binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, ) @@ -129,6 +131,7 @@ def test_ism_full_chain_split(test_info, in_fmt, bitrate, render_config, traject in_fmt=in_fmt, bitrate=bitrate, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), + binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, ) @@ -166,6 +169,7 @@ def test_masa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec in_fmt=in_fmt, bitrate=bitrate, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), + binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, ) @@ -203,6 +207,7 @@ def test_omasa_full_chain_split(test_info, in_fmt, bitrate, render_config, traje in_fmt=in_fmt, bitrate=bitrate, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), + binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, ) @@ -224,6 +229,7 @@ def test_osba_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec in_fmt=in_fmt, bitrate=bitrate, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), + binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, ) @@ -273,6 +279,7 @@ def test_full_chain_split_pcm(test_info, in_fmt, bitrate, render_config): in_fmt=in_fmt, bitrate=bitrate, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), + binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, renderer_fmt="BINAURAL_SPLIT_PCM", @@ -337,6 +344,7 @@ def test_framing_combinations_full_chain_split( pre_trajectory=pre_trajectory, bitrate="256000", post_trajectory=post_trajectory, + binary_suffix=EXE_SUFFIX, post_rend_fr=post_rend_fr, pre_rend_fr=pre_rend_fr, ) diff --git a/tests/split_rendering/utils.py b/tests/split_rendering/utils.py index d1d95a6242..b7f40cc361 100644 --- a/tests/split_rendering/utils.py +++ b/tests/split_rendering/utils.py @@ -173,14 +173,14 @@ def run_full_chain_split_rendering( tmp_dir = Path(tmp_dir) cut_in_file = tmp_dir.joinpath("cut_input.wav") #ivas_bitstream = tmp_dir.joinpath("ivas.192") - ivas_bitstream_stem = f"{in_fmt}_{bitrate}bps_{renderer_fmt}_{pre_trajectory.stem}_split_full_{post_trajectory.stem}_config_{render_config.stem}_prerfr_{pre_rend_fr}_postrfr_{post_rend_fr}_ivas.192" + ivas_bitstream_stem = f"{in_fmt}_{bitrate}bps_{renderer_fmt}_split_full_config_{render_config.stem}_prerfr_{pre_rend_fr}_postrfr_{post_rend_fr}_ivas.192" #split_bitstream = tmp_dir.joinpath("split.bit") - split_bitstream_stem = f"{in_fmt}_{bitrate}bps_{renderer_fmt}_{pre_trajectory.stem}_split_full_{post_trajectory.stem}_config_{render_config.stem}_prerfr_{pre_rend_fr}_postrfr_{post_rend_fr}_split.bit" + split_bitstream_stem = f"{in_fmt}_{bitrate}bps_{renderer_fmt}_split_full_config_{render_config.stem}_prerfr_{pre_rend_fr}_postrfr_{post_rend_fr}_split.bit" if renderer_fmt == "BINAURAL_SPLIT_PCM": #split_md_file = tmp_dir.joinpath("split_md.bin") - split_md_file_stem = f"{in_fmt}_{bitrate}bps_{renderer_fmt}_{pre_trajectory.stem}_split_full_{post_trajectory.stem}_config_{render_config.stem}_prerfr_{pre_rend_fr}_postrfr_{post_rend_fr}_split_md.bit" + split_md_file_stem = f"{in_fmt}_{bitrate}bps_{renderer_fmt}_split_full_config_{render_config.stem}_prerfr_{pre_rend_fr}_postrfr_{post_rend_fr}_split_md.bit" - out_file_stem = f"{in_fmt}_{bitrate}bps_{renderer_fmt}_{pre_trajectory.stem}_split_full_{post_trajectory.stem}_config_{render_config.stem}_prerfr_{pre_rend_fr}_postrfr_{post_rend_fr}_.wav" + out_file_stem = f"{in_fmt}_{bitrate}bps_{renderer_fmt}_split_full_config_{render_config.stem}_prerfr_{pre_rend_fr}_postrfr_{post_rend_fr}_.wav" if test_info.config.option.create_ref: output_path_base = OUTPUT_PATH_REF -- GitLab From 7a8432fb62f083a78698c78e9d0dbeed8fca09ec Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Mon, 13 May 2024 08:44:22 +0200 Subject: [PATCH 4/8] Cleanup around TEST_SET variable. Fix for HTML tags in parse_commands.py --- .gitlab-ci.yml | 13 ++++++------- scripts/parse_commands.py | 1 + 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b3359459eb..4a4404a35d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1387,12 +1387,12 @@ ivas-conformance: # Reference creation - python tests/create_short_testvectors.py - python scripts/prepare_combined_format_inputs.py - - python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files - - python -m pytest tests/split_rendering/test_split_rendering.py -v -n auto --create_ref --keep_files - - python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --update_ref 1 -m create_ref_part2 --keep_files + - $TEST_SET = "tests/codec_be_on_mr_nonselection", "tests/renderer/test_renderer.py", "tests/split_rendering/test_split_rendering.py" + - python -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files + - python -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref_part2 --keep_files # Output creation - - python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html + - python -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html - python scripts/parse_commands.py report_cmd.html Readme_IVAS.txt # Copy input data and output ref data @@ -1481,13 +1481,12 @@ ivas-conformance-linux: # Reference creation - python3 tests/create_short_testvectors.py - python3 scripts/prepare_combined_format_inputs.py - - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py" + - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py" - python3 -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files - - python3 -m pytest tests/split_rendering/test_split_rendering.py -v -n auto --create_ref --keep_files - python3 -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref_part2 --keep_files # Output creation - - python3 -m pytest $TEST_SET tests/split_rendering/test_split_rendering.py -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html + - python3 -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html - python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt # Copy input data and output ref data diff --git a/scripts/parse_commands.py b/scripts/parse_commands.py index 8b630337b5..401b5936ea 100644 --- a/scripts/parse_commands.py +++ b/scripts/parse_commands.py @@ -49,6 +49,7 @@ if __name__ == '__main__': rend_cmd = False isar_post_rend_cmd = False for line in infile.readlines(): + line = line.split("
")[0] # Remove trailing html tags if enc_cmd: cmds_enc.append(line) enc_cmd = False -- GitLab From 5c89e7123cf8342eb63a059b00eb3659a9d57526 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Mon, 13 May 2024 08:51:54 +0200 Subject: [PATCH 5/8] Fix for ivas-conformance-linux to ensure outcome of test_26252.py is reflected in job --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4a4404a35d..cc584d4623 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1545,6 +1545,9 @@ ivas-conformance-linux: - commit_sha=$(git rev-parse HEAD) - genhtml coverage.info -o coverage -t "Coverage on main @ $commit_sha" + # Check for failures + if [ $exit_code -eq 1 ]; then echo "Test failures encountered"; exit $EXIT_CODE_FAIL; fi + artifacts: name: "ivas-conformance-linux-$CI_COMMIT_SHORT_SHA" expire_in: 1 week -- GitLab From b2c65bae19366f7a4bf8441c9aeb45034d9eea04 Mon Sep 17 00:00:00 2001 From: norvell Date: Mon, 13 May 2024 06:55:20 +0000 Subject: [PATCH 6/8] Syntax fix .gitlab-ci.yml (missing '-') --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cc584d4623..be07b53f13 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1546,8 +1546,8 @@ ivas-conformance-linux: - genhtml coverage.info -o coverage -t "Coverage on main @ $commit_sha" # Check for failures - if [ $exit_code -eq 1 ]; then echo "Test failures encountered"; exit $EXIT_CODE_FAIL; fi - + - if [ $exit_code -eq 1 ]; then echo "Test failures encountered"; exit $EXIT_CODE_FAIL; fi + artifacts: name: "ivas-conformance-linux-$CI_COMMIT_SHORT_SHA" expire_in: 1 week -- GitLab From 05ca498c6e13ad65593f74503459d3628ba99002 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Mon, 13 May 2024 09:13:26 +0200 Subject: [PATCH 7/8] Add mark create_ref to split renderer tests --- tests/split_rendering/test_split_rendering.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/split_rendering/test_split_rendering.py b/tests/split_rendering/test_split_rendering.py index 3bf9db3b1c..b038d591b0 100644 --- a/tests/split_rendering/test_split_rendering.py +++ b/tests/split_rendering/test_split_rendering.py @@ -38,6 +38,7 @@ from tests.split_rendering.utils import * """ Ambisonics """ +@pytest.mark.create_ref @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_AMBI) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_AMBI) @@ -59,6 +60,7 @@ def test_ambisonics_full_chain_split( ) +@pytest.mark.create_ref @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_AMBI) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI_SPLIT_REND) @@ -78,6 +80,7 @@ def test_ambisonics_external_split(test_info, in_fmt, render_config, trajectory) """ Multichannel """ +@pytest.mark.create_ref @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MC) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_MC) @@ -99,6 +102,7 @@ def test_multichannel_full_chain_split( ) +@pytest.mark.create_ref @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MC) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC_SPLIT_REND) @@ -118,6 +122,7 @@ def test_multichannel_external_split(test_info, in_fmt, render_config, trajector """ ISM """ +@pytest.mark.create_ref @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_ISM) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_ISM) @@ -137,6 +142,7 @@ def test_ism_full_chain_split(test_info, in_fmt, bitrate, render_config, traject ) +@pytest.mark.create_ref @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_ISM) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM_SPLIT_REND) @@ -156,6 +162,7 @@ def test_ism_external_split(test_info, in_fmt, render_config, trajectory): """ MASA """ +@pytest.mark.create_ref @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MASA) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_MASA) @@ -175,6 +182,7 @@ def test_masa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec ) +@pytest.mark.create_ref @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MASA) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA_SPLIT_REND) @@ -194,6 +202,7 @@ def test_masa_external_split(test_info, in_fmt, render_config, trajectory): """ OMASA """ +@pytest.mark.create_ref @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_OMASA) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_OMASA) @@ -216,6 +225,7 @@ def test_omasa_full_chain_split(test_info, in_fmt, bitrate, render_config, traje """ OSBA """ +@pytest.mark.create_ref @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_OSBA) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_OSBA) @@ -238,6 +248,7 @@ def test_osba_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec """ PLC """ +@pytest.mark.create_ref @pytest.mark.parametrize("error_pattern", PLC_ERROR_PATTERNS) @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_PLC) @@ -268,6 +279,7 @@ full_chain_split_pcm_params = [ ] +@pytest.mark.create_ref @pytest.mark.parametrize("in_fmt,bitrate,render_config", full_chain_split_pcm_params) def test_full_chain_split_pcm(test_info, in_fmt, bitrate, render_config): trajectory = SPLIT_REND_HR_TRAJECTORIES_TO_TEST[0] @@ -293,6 +305,7 @@ external_split_pcm_params = [ ] +@pytest.mark.create_ref @pytest.mark.parametrize("in_fmt,render_config", external_split_pcm_params) def test_external_split_pcm(test_info, in_fmt, render_config): trajectory = SPLIT_REND_HR_TRAJECTORIES_TO_TEST[0] @@ -308,6 +321,7 @@ def test_external_split_pcm(test_info, in_fmt, render_config): renderer_fmt="BINAURAL_SPLIT_PCM", ) +@pytest.mark.create_ref @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_FRAMING) @pytest.mark.parametrize("in_fmt", ["5_1"]) @@ -326,6 +340,7 @@ def test_framing_combinations_external_split(test_info, in_fmt, render_config, t post_rend_fr=post_rend_fr, pre_rend_fr=pre_rend_fr, ) +@pytest.mark.create_ref @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_FRAMING) @pytest.mark.parametrize("in_fmt", ["5_1"]) -- GitLab From a14948975314904846bf295a213d90331a809f0b Mon Sep 17 00:00:00 2001 From: rtyag Date: Mon, 13 May 2024 17:50:44 +1000 Subject: [PATCH 8/8] reduce filename size for external renderer tests as well --- tests/split_rendering/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/split_rendering/utils.py b/tests/split_rendering/utils.py index b7f40cc361..0c5748b0ee 100644 --- a/tests/split_rendering/utils.py +++ b/tests/split_rendering/utils.py @@ -305,7 +305,7 @@ def run_external_split_rendering( split_bitstream = tmp_dir.joinpath("split.bit") if renderer_fmt == "BINAURAL_SPLIT_PCM": split_md_file = tmp_dir.joinpath("split_md.bin") - out_file_stem = f"{in_fmt}_{renderer_fmt}_{pre_trajectory.stem}_split_ext_{post_trajectory.stem}_config_{render_config.stem}_postrfr_{pre_rend_fr}_prerfr_{post_rend_fr}.wav" + out_file_stem = f"{in_fmt}_{renderer_fmt}_split_ext_config_{render_config.stem}_postrfr_{pre_rend_fr}_prerfr_{post_rend_fr}.wav" if test_info.config.option.create_ref: output_path_base = OUTPUT_PATH_REF -- GitLab