diff --git a/ci/basop-pages/create_summary_page.py b/ci/basop-pages/create_summary_page.py index 1f146c4cbbb5bfacf57bb908958ee935b3051d3e..f8ddb00ae8fb23f38bc55315e1dae05fd7d6e024 100644 --- a/ci/basop-pages/create_summary_page.py +++ b/ci/basop-pages/create_summary_page.py @@ -2,7 +2,7 @@ import argparse from create_report_pages import SUBPAGE_TMPL_CSS, FORMATS -MEASURES = ["MLD","DIFF","SSNR","ODG"] +MEASURES = ["MLD","DIFF","SSNR","ODG","DELTA_ODG"] SUMMARY_PAGE_TMPL_HTML = """ diff --git a/ci/remove_unsupported_testcases.py b/ci/remove_unsupported_testcases.py index b316a417e5a76122fecd7826f54522cfabc818b8..7eb13492dc1357dbe4bd500c3cf86213e454f4cc 100644 --- a/ci/remove_unsupported_testcases.py +++ b/ci/remove_unsupported_testcases.py @@ -66,6 +66,23 @@ TESTCASES = [ "MASA 1TC at 256kbps, 48kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out custom configuration", "MASA 1TC at 256 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out, HR custom configuration", "OMASA 2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out", + "OMASA 2Dir2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM", + "OMASA 2Dir2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC", + "OMASA 2Dir2TC 2ISM at 96 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC", + "OMASA 2Dir2TC 2ISM br sw techs 13.2 to 512 kbps start 48 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing", + "OSBA 3OA 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, DISC", + "OSBA 2OA 3ISM at 128 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC", + "OSBA 2OA 3ISM at bitrate switching 13.2 to 512 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing", + "4 ISM with metadata at 32 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, PARAM_ISM", + "4 ISM with metadata at 128 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, DISC", + "3 ISM with metadata at 24.4 kbps, 48 kHz in, 48 kHz out, FOA out, object editing, JBM Prof 5, PARAM_ISM", + "3 ISM with metadata at 384 kbps, 48 kHz in, 48 kHz out, FOA out, object editing, JBM Prof 5, DISC", + "4 ISM with metadata bitrate switching from 32 kbps to 48 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_IR out, object editing", + "4 ISM with metadata bitrate switching from 48 kbps to 32 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, JBM Prof 5", + "OMASA 2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM", + "OMASA 2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC", + "OMASA 2TC 2ISM at 96 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC", + "OMASA 2TC 2ISM br sw techs 13.2 to 512 kbps start 48 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing", ] diff --git a/scripts/create_histogram_summary.py b/scripts/create_histogram_summary.py index 0b10b76b8d2ba3c2aebcabd9927276b4e7245c3c..af9a11de2ab48e10f897301da566b826ca06c0c7 100644 --- a/scripts/create_histogram_summary.py +++ b/scripts/create_histogram_summary.py @@ -75,6 +75,7 @@ if __name__ == "__main__": "DIFF": ("MAXIMUM ABS DIFF", None), "SSNR": ("MIN_SSNR", None), "ODG": ("MIN_ODG", None), + "DELTA_ODG": ("DELTA_ODG", None), } else: limits_per_measure = { @@ -82,6 +83,7 @@ if __name__ == "__main__": "DIFF": ("MAXIMUM ABS DIFF", [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769]), "SSNR": ("MIN_SSNR", [-math.inf, 0, 10, 20, 30, 40, 40, 50, 60, 100]), "ODG": ("MIN_ODG", [-5, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5]), + "DELTA_ODG": ("DELTA_ODG", [-5, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5]), } (measure_label, limits) = limits_per_measure[measure] diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index d68637c64977f7538957ce1b0cb3d6c47c311e13..1f2157ca8f5b6dc30cdd53e9bf080b9a0e8afba3 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -72,6 +72,11 @@ if __name__ == "__main__": action="store_true", help="Extract clipping information. Available if encoder has been run with DEBUGGING active.", ) + parser.add_argument( + "--delta_odg", + action="store_true", + help="Extract Delta ODG information.", + ) parser.add_argument( "--skip_formats", action="store_true", @@ -90,6 +95,8 @@ if __name__ == "__main__": CATEGORIES = IVAS_CATEGORIES if args.clipping: PROPERTIES += ["ENC_CORE_OVL","MAX_OVL","MIN_OVL"] + if args.delta_odg: + PROPERTIES += ["DELTA_ODG"] if args.skip_formats: FORMATS = NO_FORMATS CATEGORIES = NO_CATEGORIES diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index c19bf6a6d9a85d9f7464c84e7e5aec12a4951545..cb779ff51765cbdfa91e4eb75005b99513dac29c 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -34,6 +34,7 @@ def cmp_pcm( abs_tol=0, get_ssnr=False, get_odg=False, + get_odg_bin=False, odg_input=None, odg_test=None, odg_ref=None, @@ -133,6 +134,16 @@ def cmp_pcm( reason += msg + " - " if get_odg: + for n in range(nchannels): + pqeval_output = pqevalaudio_wrapper(s1[:, n], s2[:, n], fs) + + match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) + odg = float(match_odg.groups()[0]) + msg = f"Channel {n} ODG: {odg}" + reason += " - " + msg + print(msg) + + if get_odg_bin: odg_files = {} for f in [odg_input, odg_test, odg_ref]: # Load PEAQ test files and ensure 48 kHz sampling rate @@ -165,7 +176,7 @@ def cmp_pcm( odg = odg_test - odg_ref # Todo: store both rather than difference? - msg = f"ODG: {odg}" + msg = f"Delta-ODG: {odg}" reason += " - " + msg print(msg, file=output_target) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 98c54b7cdb9d25c54256cfe9e2f66490bb57a5b9..e760edd4702fd68859a1c309d31a543a21737d70 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -217,6 +217,7 @@ def test_param_file_tests( get_ssnr, get_enc_stats, get_odg, + get_odg_bin, compare_to_input, compare_enc_dmx, ): @@ -248,6 +249,7 @@ def test_param_file_tests( get_ssnr, get_enc_stats, get_odg, + get_odg_bin, compare_to_input, compare_enc_dmx, ) @@ -279,6 +281,7 @@ def run_test( get_ssnr, get_enc_stats, get_odg, + get_odg_bin, compare_to_input, compare_enc_dmx, ): @@ -526,7 +529,7 @@ def run_test( odg_input = None odg_test = None odg_ref = None - if get_odg: + if get_odg_bin: # Find input format in_fmt = [(a, b) for (a, b) in INPUT_FMT if re.search(a, enc_opts)][0][1] @@ -595,6 +598,7 @@ def run_test( allow_differing_lengths=allow_differing_lengths, get_ssnr=get_ssnr, get_odg=get_odg, + get_odg_bin=get_odg_bin, odg_input=odg_input, odg_test=odg_test, odg_ref=odg_ref, diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index a0c72b64f020ba9f4da4cac630035b496a86a886..55c7dcf3fe884712edd4ed47078833fd3713b3f2 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -112,6 +112,7 @@ def test_pca_enc( abs_tol, get_ssnr, get_odg, + get_odg_bin, get_enc_stats, compare_to_input, compare_enc_dmx, @@ -195,6 +196,7 @@ def test_pca_enc( abs_tol=abs_tol, get_ssnr=get_ssnr, get_odg=get_odg, + get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, ) @@ -231,6 +233,7 @@ def test_sba_enc_system( abs_tol, get_ssnr, get_odg, + get_odg_bin, get_enc_stats, compare_to_input, compare_enc_dmx, @@ -372,6 +375,7 @@ def test_sba_enc_system( abs_tol=abs_tol, get_ssnr=get_ssnr, get_odg=get_odg, + get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, ) @@ -400,6 +404,7 @@ def test_spar_hoa2_enc_system( abs_tol, get_ssnr, get_odg, + get_odg_bin, get_enc_stats, compare_to_input, compare_enc_dmx, @@ -516,6 +521,7 @@ def test_spar_hoa2_enc_system( abs_tol=abs_tol, get_ssnr=get_ssnr, get_odg=get_odg, + get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, ) @@ -544,6 +550,7 @@ def test_spar_hoa3_enc_system( abs_tol, get_ssnr, get_odg, + get_odg_bin, get_enc_stats, compare_to_input, compare_enc_dmx, @@ -654,6 +661,7 @@ def test_spar_hoa3_enc_system( abs_tol=abs_tol, get_ssnr=get_ssnr, get_odg=get_odg, + get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, ) @@ -686,6 +694,7 @@ def test_sba_enc_BWforce_system( abs_tol, get_ssnr, get_odg, + get_odg_bin, get_enc_stats, compare_to_input, compare_enc_dmx, @@ -812,6 +821,7 @@ def test_sba_enc_BWforce_system( abs_tol=abs_tol, get_ssnr=get_ssnr, get_odg=get_odg, + get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, ) @@ -848,6 +858,7 @@ def test_sba_plc_system( abs_tol, get_ssnr, get_odg, + get_odg_bin, get_enc_stats, compare_to_input, compare_enc_dmx, @@ -951,6 +962,7 @@ def test_sba_plc_system( abs_tol=abs_tol, get_ssnr=get_ssnr, get_odg=get_odg, + get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, ) @@ -1126,6 +1138,7 @@ def sba_dec( abs_tol=0, get_ssnr=False, get_odg=False, + get_odg_bin=False, compare_to_input=False, ): dut_pkt_dir = f"{dut_base_path}/sba_bs/pkt" @@ -1198,7 +1211,7 @@ def sba_dec( odg_input = None odg_test = None odg_ref = None - if get_odg: + if get_odg_bin: enc_opts = "" dec_opts = "" input_file = f"{test_vector_path}/{tag}.wav" @@ -1249,6 +1262,7 @@ def sba_dec( allow_differing_lengths=allow_differing_lengths, get_ssnr=get_ssnr, get_odg=get_odg, + get_odg_bin=get_odg_bin, odg_input=odg_input, odg_test=odg_test, odg_ref=odg_ref, diff --git a/tests/conftest.py b/tests/conftest.py index 1fd548641756c62ec2caca1954059152ebc95f2f..91b9f786bd7bf962b0e6f4bd4e8782184f8faa62 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -59,10 +59,12 @@ from .constants import ( SSNR_PATTERN, ENC_AUX_FILES, ODG_PATTERN, + DELTA_ODG_PATTERN, MLD, MAX_ABS_DIFF, SSNR, ODG, + DELTA_ODG, MAX_ENC_DIFF, MAX_ENC_DIFF_PATTERN, ENC_CORE_OVL, @@ -222,6 +224,12 @@ def pytest_addoption(parser): help="Get Objective Difference Grade for both conditions during comparison and report difference", ) + parser.addoption( + "--odg_bin", + action="store_true", + help="Get Objective Difference Grade for both conditions during comparison and report Delta ODG for binauralized signals", + ) + parser.addoption( "--create_ref", action="store_true", @@ -350,6 +358,13 @@ def get_odg(request): """ return request.config.option.odg +@pytest.fixture(scope="session", autouse=True) +def get_odg_bin(request): + """ + Return indication to compute Delta PEAQ ODG during ref/dut comparison. Uses binauralization of input and output signals. + """ + return request.config.option.odg_bin + @pytest.fixture(scope="session") def abs_tol(request) -> int: @@ -1110,7 +1125,7 @@ def pytest_configure(config): @pytest.fixture(scope="session") def props_to_record( - request, get_mld, get_ssnr, get_odg, get_enc_stats, encoder_only + request, get_mld, get_ssnr, get_odg, get_odg_bin, get_enc_stats, encoder_only ) -> str: props = [] @@ -1125,6 +1140,8 @@ def props_to_record( props.append(SSNR) if get_odg: props.append(ODG) + if get_odg_bin: + props.append(DELTA_ODG) return props @@ -1170,6 +1187,10 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: max_enc_diff_param_name, _, max_enc_diff_ratio = search_result.groups(0) props[MAX_ENC_DIFF] = float(max_enc_diff_ratio) props[MAX_ENC_DIFF_PARAM_NAME] = max_enc_diff_param_name + elif prop == DELTA_ODG: + delta_odg = re.search(DELTA_ODG_PATTERN, text_to_parse) + if delta_odg: + props["DELTA_ODG"] = delta_odg.groups(1)[0] return props diff --git a/tests/constants.py b/tests/constants.py index 865a7061a7fdab6d2e21eb14f0e317ad58fd1cd8..9990db37cc690d2a52eb983df927740eacf5b46b 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -11,6 +11,7 @@ MLD = "MLD" MAX_ABS_DIFF = "MAXIMUM ABS DIFF" SSNR = "SSNR" ODG = "ODG" +DELTA_ODG = "Delta-ODG" MAX_ENC_DIFF = "MAXIMUM ENC DIFF" MAX_ENC_DIFF_PARAM_NAME = "MAXIMUM ENC DIFF PARAM" ENC_CORE_OVL = "ENC_CORE_OVL" @@ -24,7 +25,8 @@ DMX_SSNR = "DMX_SSNR" MLD_PATTERN = r"MLD: ([\d\.]*)" MAX_DIFF_PATTERN = r"MAXIMUM ABS DIFF: (\d*)" ODG_PATTERN_PQEVALAUDIO = r"Objective Difference Grade: (-*\d*\.\d*)" -ODG_PATTERN = r"ODG: (-*\d*\.\d*)" +ODG_PATTERN = r"(?