diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index e3557e252ba32c9b33962f0f7cad677d05d1624d..c849b160be964c75935ffa4e79e858249545e479 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -92,11 +92,12 @@ ARROW_DOWN = '' # expected columns. actual columns are filtered from the incoming data later, this # is mainly for controlling the order in the output table -COLUMNS = ["testcase", "Result", "MLD", "MAXIMUM ABS DIFF"] +COLUMNS = ["testcase", "Result", "MLD", "MAXIMUM ABS DIFF", "MIN_SSNR"] COLUMNS_GLOBAL = COLUMNS[:1] COLUMNS_DIFFERENTIAL = COLUMNS[1:] COLUMNS_DIFFERENTIAL_NOT_MLD = COLUMNS_DIFFERENTIAL[2:] + def create_subpage( html_out, csv_out, @@ -111,11 +112,18 @@ def create_subpage( ) write_out_csv(merged_reports, merged_reports[0].keys(), csv_out) - table_header_a = "".join([TH_TMPL_GLOBAL.format(c) for c in COLUMNS_GLOBAL] + [TH_TMPL_DIFFERENTIAL.format(c) for c in COLUMNS_DIFFERENTIAL]) + table_header_a = "".join( + [TH_TMPL_GLOBAL.format(c) for c in COLUMNS_GLOBAL] + + [TH_TMPL_DIFFERENTIAL.format(c) for c in COLUMNS_DIFFERENTIAL] + ) table_header_b = list() for c in COLUMNS_DIFFERENTIAL: - table_header_b.append(TH_TMPL_SECOND_ROW.format(f"Previous Run
ID: {id_previous}")) - table_header_b.append(TH_TMPL_SECOND_ROW.format(f"Current Run
ID: {id_current}")) + table_header_b.append( + TH_TMPL_SECOND_ROW.format(f"Previous Run
ID: {id_previous}") + ) + table_header_b.append( + TH_TMPL_SECOND_ROW.format(f"Current Run
ID: {id_current}") + ) table_header_b = "".join(table_header_b) table_body = "\n".join( tr_from_row(row, id_current, id_previous) for row in merged_reports @@ -241,8 +249,13 @@ def merge_and_cleanup_mld_reports( return diff - other_col_pairs = [(f"{col}-{id_previous}", f"{col}-{id_current}") for col in COLUMNS_DIFFERENTIAL_NOT_MLD] - merged = sorted(merged, key=partial(sort_func, other_col_pairs=other_col_pairs), reverse=True) + other_col_pairs = [ + (f"{col}-{id_previous}", f"{col}-{id_current}") + for col in COLUMNS_DIFFERENTIAL_NOT_MLD + ] + merged = sorted( + merged, key=partial(sort_func, other_col_pairs=other_col_pairs), reverse=True + ) # remove the unecessary whole path from the testcase names for row in merged: diff --git a/scripts/parse_mld_xml.py b/scripts/parse_mld_xml.py index 7370cfe7ff3b39e2bb2319034b06880c77984070..a7f10f94a468968934337438230d0d5d08c2bbe8 100644 --- a/scripts/parse_mld_xml.py +++ b/scripts/parse_mld_xml.py @@ -7,7 +7,7 @@ from xml.etree import ElementTree Parse a junit report and create an MLD summary report. """ -PROPERTIES = ["MLD", "MAXIMUM ABS DIFF"] +PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR"] # Main routine diff --git a/scripts/pyaudio3dtools/audioarray.py b/scripts/pyaudio3dtools/audioarray.py index d89c079d8989d87841c8d95ac2d425639b92d7ca..0c313c4c91da056c46df90bba2a43361d6140da3 100644 --- a/scripts/pyaudio3dtools/audioarray.py +++ b/scripts/pyaudio3dtools/audioarray.py @@ -1,36 +1,37 @@ #!/usr/bin/env python3 """ - (C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +(C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ import logging +import warnings import math import multiprocessing as mp import platform @@ -232,6 +233,10 @@ def compare( fs: int, per_frame: bool = True, get_mld: bool = False, + get_ssnr: bool = False, + ssnr_thresh_low: float = -np.inf, + ssnr_thresh_high: float = np.inf, + apply_thresholds_to_ref_only: bool = False, ) -> dict: """Compare two audio arrays @@ -247,6 +252,18 @@ def compare( Compute difference per frame (default True) get_mld: bool Run MLD tool if there is a difference between the signals (default False) + get_ssnr: bool + Compute Segmental SNR between signals + ssnr_thresh_low: float + Low threshold for including a segment in the SSNR computation. Per default, both + reference and test signal power are compared to this threshold, see below + ssnr_thresh_high: float + High threshold for including a segment in the SSNR computation. Per default, both + reference and test signal power are compared to this threshold, see below + apply_thresholds_to_ref_only: bool + Set to True to only apply the threshold comparison for the reference signal + for whether to include a segment in the ssnr computation. Use this to align + behaviour with the MPEG-D conformance specification. Returns ------- @@ -266,8 +283,13 @@ def compare( "first_diff_pos_sample": -1, "first_diff_pos_channel": -1, "first_diff_pos_frame": -1, - "MLD": 0 if get_mld else None, } + + if get_mld: + result["MLD"] = 0 + if get_ssnr: + result["SSNR"] = np.asarray([np.inf] * ref.shape[1]) + if per_frame: result["max_abs_diff_pos_frame"] = 0 result["nframes_diff"] = 0 @@ -320,7 +342,6 @@ def compare( result["nframes_diff_percentage"] = nframes_diff_percentage if get_mld: - mld_max = 0 toolsdir = Path(__file__).parent.parent.joinpath("tools") if platform.system() == "Windows": @@ -334,8 +355,14 @@ def compare( for i in range(nchannels): tmpfile_ref = Path(tmpdir).joinpath(f"ref_ch{i+1}.wav") tmpfile_test = Path(tmpdir).joinpath(f"test_ch{i+1}.wav") - r48 = np.clip( resample(ref[:, i].astype(float), fs, 48000), -32768, 32767 ).astype(np.int16) # Convert to float for resample, then to int16 for wavfile.write - t48 = np.clip( resample(test[:, i].astype(float), fs, 48000), -32768, 32767 ).astype(np.int16) + r48 = np.clip( + resample(ref[:, i].astype(float), fs, 48000), -32768, 32767 + ).astype( + np.int16 + ) # Convert to float for resample, then to int16 for wavfile.write + t48 = np.clip( + resample(test[:, i].astype(float), fs, 48000), -32768, 32767 + ).astype(np.int16) wavfile.write(str(tmpfile_ref), 48000, r48) wavfile.write(str(tmpfile_test), 48000, t48) out = subprocess.check_output([mld, tmpfile_ref, tmpfile_test]) @@ -343,6 +370,19 @@ def compare( result["MLD"] = mld_max + if get_ssnr: + # length of segment is always 20ms + len_seg = int(0.02 * fs) + print(len_seg, ref.shape, test.shape) + result["SSNR"] = ssnr( + ref, + test, + len_seg, + thresh_low=ssnr_thresh_low, + thresh_high=ssnr_thresh_high, + apply_thresholds_to_ref_only=apply_thresholds_to_ref_only, + ) + return result @@ -513,3 +553,75 @@ def process_async(files: Iterable, func: Callable, **kwargs): for r in results: r.get() return results + + +def ssnr( + ref_sig: np.ndarray, + test_sig: np.ndarray, + len_seg: int, + thresh_low: float = -200, + thresh_high: float = 0, + apply_thresholds_to_ref_only: bool = False, +) -> np.ndarray: + """ + Calculate Segmental SNR for test_sig to ref_sig as defined in ISO/IEC 14496-4 + """ + ss = list() + + ref_sig_norm = ref_sig / -np.iinfo(np.int16).min + test_sig_norm = test_sig / -np.iinfo(np.int16).min + + # check if diff of signal is zero already, then SNR is infinite, since no noise + diff_sig_norm = ref_sig_norm - test_sig_norm + if np.all(diff_sig_norm == 0): + return np.asarray([np.inf] * ref_sig_norm.shape[1]) + + channels_identical_idx = np.sum(np.abs(diff_sig_norm), axis=0) == 0 + + denom_add = 10**-13 * len_seg + segment_counter = np.zeros(ref_sig.shape[1]) + + # iterate over test signal too to allow power comparison to threshold + for ref_seg, diff_seg, test_seg in zip( + get_framewise(ref_sig_norm, len_seg, zero_pad=True), + get_framewise(diff_sig_norm, len_seg, zero_pad=True), + get_framewise(test_sig_norm, len_seg, zero_pad=True), + ): + nrg_ref = np.sum(ref_seg**2, axis=0) + nrg_diff = np.sum(diff_seg**2, axis=0) + + ss_seg = np.log10(1 + nrg_ref / (denom_add + nrg_diff)) + + # only sum up segments that fall inside the thresholds + # add small eps to nrg_ref to prevent RuntimeWarnings from numpy + ref_power = 10 * np.log10((nrg_ref + 10**-7) / len_seg) + zero_mask = np.logical_or(ref_power < thresh_low, ref_power > thresh_high) + + # create same mask for test signal + if not apply_thresholds_to_ref_only: + nrg_test = np.sum(test_seg**2, axis=0) + test_power = 10 * np.log10((nrg_test + 10**-7) / len_seg) + zero_mask_test = np.logical_or( + test_power < thresh_low, test_power > thresh_high + ) + zero_mask = np.logical_or(zero_mask, zero_mask_test) + + ss_seg[zero_mask] = 0 + # increase segment counter only for channels that were not zeroed + segment_counter += np.logical_not(zero_mask) + + ss.append(ss_seg) + + # if the reference signal was outside the thresholds for all segments in a channel, segment_counter is zero + # for that channel and the division here would trigger a warning. We supress the warning and later + # set the SSNR for those channels to nan manually instead (overwriting later is simply easier than adding ifs here) + with warnings.catch_warnings(): + ssnr = np.round( + 10 * np.log10(10 ** (np.sum(ss, axis=0) / segment_counter) - 1), 2 + ) + ssnr[segment_counter == 0] = np.nan + + # this prevents all-zero channels in both signals to be reported as -inf + ssnr[channels_identical_idx] = np.inf + + return ssnr diff --git a/scripts/ssnr.py b/scripts/ssnr.py new file mode 100644 index 0000000000000000000000000000000000000000..89ac4c6bf999adf8a9530e718a94651e2ba9ac52 --- /dev/null +++ b/scripts/ssnr.py @@ -0,0 +1,62 @@ +import argparse +import sys +import pathlib +from pyaudio3dtools import audiofile, audioarray + + +def main(args): + ref_sig, fs_ref = audiofile.readfile(args.ref_file) + test_sig, fs_test = audiofile.readfile(args.test_file) + + if fs_ref != fs_test: + print("Files need to have same sampling rate!") + return -1 + + len_seg = int(20 * fs_ref / 1000) + print(len_seg, ref_sig.shape, test_sig.shape) + ssnr = audioarray.ssnr( + ref_sig, + test_sig, + len_seg, + args.thresh_low, + args.thresh_high, + args.apply_thresholds_on_ref_only, + ) + + for i, s in enumerate(ssnr, start=1): + print(f"Channel {i}: {s}") + + return 0 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("ref_file", type=pathlib.Path, help="Reference signal wav file") + parser.add_argument( + "test_file", type=pathlib.Path, help="Signal under test wav file" + ) + parser.add_argument( + "--thresh_low", + type=float, + default="-inf", + help="Low threshold for signal power in a segment to be used in the SSNR calculation (default: -inf).\n" + "Applied to both signals per default (see apply_thresholds_on_ref_only argument).", + ) + parser.add_argument( + "--thresh_high", + type=float, + default="inf", + help="High threshold for signal power in a segment to be used in the SSNR calculation (default: +inf).\n" + "Applied to both signals per default (see apply_thresholds_on_ref_only argument).", + ) + parser.add_argument( + "--apply_thresholds_on_ref_only", + action="store_true", + default=False, + help="Use this to apply the thresholding on signal power to the reference signal only.\n" + "This makes the implementation behaviour conform to the MPEG-D conformance spec.", + ) + + args = parser.parse_args() + + sys.exit(main(args)) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 9783e8b5613cfe89a5ba054b95e3d9cfb5f5c68e..7bf1359f4e9fd7249278638e78138bbc64f008c8 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -13,14 +13,15 @@ import pyivastest def cmp_pcm( - file1, - file2, + ref_file, + cmp_file, out_config, fs, get_mld=False, allow_differing_lengths=False, mld_lim=0, abs_tol=0, + get_ssnr=False, ) -> (int, str): """ Compare 2 PCM files for bitexactness @@ -39,8 +40,12 @@ def cmp_pcm( else: nchannels = pyivastest.constants.OC_TO_NCHANNELS[out_config.upper()] - s1, _ = pyaudio3dtools.audiofile.readfile(file1, nchannels, fs, outdtype=np.int16) - s2, _ = pyaudio3dtools.audiofile.readfile(file2, nchannels, fs, outdtype=np.int16) + s1, _ = pyaudio3dtools.audiofile.readfile( + ref_file, nchannels, fs, outdtype=np.int16 + ) + s2, _ = pyaudio3dtools.audiofile.readfile( + cmp_file, nchannels, fs, outdtype=np.int16 + ) # In case of wav input, override the nchannels with the one from the wav header nchannels = s1.shape[1] @@ -62,7 +67,13 @@ def cmp_pcm( return 1, reason cmp_result = pyaudio3dtools.audioarray.compare( - s1, s2, fs, per_frame=False, get_mld=get_mld + s1, + s2, + fs, + per_frame=False, + get_mld=get_mld, + get_ssnr=get_ssnr, + ssnr_thresh_low=-50, ) output_differs = 0 @@ -90,13 +101,20 @@ def cmp_pcm( else: reason += f" > {mld_lim}" + if get_ssnr: + reason += "\n" + for i, s in enumerate(cmp_result["SSNR"], start=1): + msg = f"Channel {i} SSNR: {s}" + reason += msg + "\n" + print(msg) + return output_differs, reason if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("file1", type=str) - parser.add_argument("file2", type=str) + parser.add_argument("ref_file", type=str) + parser.add_argument("cmp_file", type=str) parser.add_argument( "-o", "--out_config", diff --git a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py index 34fa40e278234b784a8c8e9fdf63aac78cdbb3b3..0a2ef6c30b4a1d14dc04e14c6fcdd8cf1c44d31d 100644 --- a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py +++ b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py @@ -35,16 +35,13 @@ __doc__ = """ import errno import os -import re from filecmp import cmp from typing import Optional import pytest from tests.cmp_pcm import cmp_pcm -from tests.conftest import DecoderFrontend, EncoderFrontend - -from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN +from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties # params # output_mode_list = ['MONO', 'STEREO', '5_1', '7_1', '5_1_2', '5_1_4', '7_1_4', 'FOA', 'HOA2', 'HOA3', 'BINAURAL', 'BINAURAL_ROOM', 'EXT'] @@ -98,6 +95,7 @@ def check_and_makedir(dir_path): ) def test_masa_enc_dec( record_property, + props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, ref_encoder_frontend: EncoderFrontend, @@ -114,6 +112,7 @@ def test_masa_enc_dec( get_mld_lim, decoder_only, abs_tol, + get_ssnr, ): # Input parameters in_fs = 48 @@ -206,71 +205,39 @@ def test_masa_enc_dec( ) # Compare outputs. For EXT output, also compare metadata. + metacmp_res = True if output_mode == "EXT": # Compare metadata as binary blob metacmp_res = cmp(dec_met_output_ref, dec_met_output_dut) - # Compare audio outputs - pcmcmp_res, reason = cmp_pcm( - dec_output_dut, - dec_output_ref, - output_mode, - int(out_fs * 1000), - get_mld=get_mld, - mld_lim=get_mld_lim, - abs_tol=abs_tol, - ) - if get_mld: - mld = re.search(MLD_PATTERN, reason).groups(1)[0] - record_property("MLD", mld) - - max_diff = 0 - if pcmcmp_res: - search_result = re.search(MAX_DIFF_PATTERN, reason) - max_diff = search_result.groups(1)[0] - record_property("MAXIMUM ABS DIFF", max_diff) - - if get_mld and get_mld_lim > 0: - if pcmcmp_res != 0: - pytest.fail(reason) - else: - # Fail if compare fails compare result - if metacmp_res == False and pcmcmp_res != 0: - pytest.fail("Metadata and transport output difference detected") - elif metacmp_res == False: - pytest.fail("Metadata output difference detected") - elif pcmcmp_res != 0: - pytest.fail("Transport output difference detected") - else: - print("Comparison bit exact") - - else: - # Compare audio outputs - filecmp_res = cmp(dec_output_ref, dec_output_dut) - if filecmp_res == False: - cmp_result, reason = cmp_pcm( - dec_output_dut, - dec_output_ref, - output_mode, - int(out_fs * 1000), - get_mld=get_mld, - mld_lim=get_mld_lim, - abs_tol=abs_tol, - ) - if get_mld: - mld = re.search(MLD_PATTERN, reason).groups(1)[0] - record_property("MLD", mld) + # Compare audio outputs + pcmcmp_res, reason = cmp_pcm( + dec_output_dut, + dec_output_ref, + output_mode, + int(out_fs * 1000), + get_mld=get_mld, + mld_lim=get_mld_lim, + abs_tol=abs_tol, + get_ssnr=get_ssnr + ) - search_result = re.search(MAX_DIFF_PATTERN, reason) - max_diff = search_result.groups(1)[0] - record_property("MAXIMUM ABS DIFF", max_diff) + props = parse_properties(reason, pcmcmp_res != 0, props_to_record) + for k, v in props.items(): + record_property(k, v) - # Report compare result - if cmp_result != 0: - pytest.fail(reason) + if get_mld and get_mld_lim > 0: + if pcmcmp_res != 0: + pytest.fail(reason) + else: + # Fail if compare fails compare result + if not metacmp_res and pcmcmp_res != 0: + pytest.fail("Metadata and transport output difference detected") + elif not metacmp_res: + pytest.fail("Metadata output difference detected") + elif pcmcmp_res != 0: + pytest.fail("Transport output difference detected") else: - if get_mld: - record_property("MLD", "0") print("Comparison bit exact") diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index eee9fcd9b274eaadbff8e1c21f222ca397a04458..814a6441df7a74d55f8344b10962758ebafc395f 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -35,7 +35,6 @@ Execute tests specified via a parameter file. import errno import filecmp import os -import re import platform from pathlib import Path from subprocess import run @@ -43,9 +42,9 @@ import pytest import numpy as np from tests.cmp_pcm import cmp_pcm -from tests.conftest import DecoderFrontend, EncoderFrontend +from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties from tests.testconfig import PARAM_FILE -from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN + VALID_DEC_OUTPUT_CONF = [ "MONO", @@ -136,6 +135,7 @@ def convert_test_string_to_tag(test_string): @pytest.mark.parametrize("param_file_id", [PARAM_FILE_ID]) def test_param_file_tests( record_property, + props_to_record, decoder_only, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, @@ -152,6 +152,7 @@ def test_param_file_tests( get_mld, get_mld_lim, abs_tol, + get_ssnr, ): enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag] @@ -341,31 +342,21 @@ def test_param_file_tests( fs = int(sampling_rate) * 1000 output_differs, reason = cmp_pcm( - dut_output_file, ref_output_file, + dut_output_file, output_config, fs, get_mld=get_mld, mld_lim=get_mld_lim, abs_tol=abs_tol, allow_differing_lengths=allow_differing_lengths, + get_ssnr=get_ssnr, ) md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config) - if get_mld: - mld = re.search(MLD_PATTERN, reason).groups(1)[0] - record_property("MLD", mld) - - max_diff = 0 - if output_differs: - search_result = re.search(MAX_DIFF_PATTERN, reason) - if search_result: - max_diff = search_result.groups(1)[0] - else: - msg = "Error " + MAX_DIFF_PATTERN + " not found" - print(msg) - pytest.fail(msg) - record_property("MAXIMUM ABS DIFF", max_diff) + props = parse_properties(reason, output_differs, props_to_record) + for k, v in props.items(): + record_property(k, v) metadata_differs = False for md_file in md_out_files: diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py index 289d10fae131b4997c437ba9eff3e54bc11515ae..e188d4a11454ff08769f4477a2dcec76ff63af59 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py @@ -34,13 +34,12 @@ __doc__ = """ import errno import os -import re import pytest from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend -from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN +from ..conftest import parse_properties # params tag_list = ["stvFOA"] @@ -77,6 +76,7 @@ def check_and_makedir(dir_path): @pytest.mark.parametrize("gain_flag", gain_list) def test_sba_plc_system( record_property, + props_to_record, dut_decoder_frontend: DecoderFrontend, test_vector_path, reference_path, @@ -93,6 +93,7 @@ def test_sba_plc_system( get_mld, get_mld_lim, abs_tol, + get_ssnr, ): SID = 0 if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]: @@ -115,6 +116,7 @@ def test_sba_plc_system( # dec sba_dec_plc( record_property, + props_to_record, dut_decoder_frontend, test_vector_path, reference_path, @@ -132,6 +134,7 @@ def test_sba_plc_system( get_mld=get_mld, get_mld_lim=get_mld_lim, abs_tol=abs_tol, + get_ssnr=get_ssnr, ) @@ -139,6 +142,7 @@ def test_sba_plc_system( # -------------------- test function -------------------- def sba_dec_plc( record_property, + props_to_record, decoder_frontend, test_vector_path, reference_path, @@ -156,6 +160,7 @@ def sba_dec_plc( get_mld=False, get_mld_lim=0, abs_tol=0, + get_ssnr=False, ): # ------------ run cmd ------------ @@ -213,16 +218,12 @@ def sba_dec_plc( get_mld=get_mld, mld_lim=get_mld_lim, abs_tol=abs_tol, + get_ssnr=get_ssnr, ) - if get_mld: - mld = re.search(MLD_PATTERN, reason).groups(1)[0] - record_property("MLD", mld) - - max_diff = 0 - if cmp_result: - search_result = re.search(MAX_DIFF_PATTERN, reason) - max_diff = search_result.groups(1)[0] - record_property("MAXIMUM ABS DIFF", max_diff) + + props = parse_properties(reason, cmp_result!=0, props_to_record) + for k, v in props.items(): + record_property(k, v) # report compare result if cmp_result != 0: diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index 2415d9d9a9cae225bcfc03ae6a274624cf874c74..ce99c55a272c9fa69459d34bea7903e126586965 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -35,14 +35,13 @@ __doc__ = """ import errno import os -import re import pytest from cut_bs import cut_from_start from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend -from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN +from ..conftest import parse_properties # params @@ -92,6 +91,7 @@ def check_and_makedir(dir_path): @pytest.mark.parametrize("fs", sample_rate_list) def test_pca_enc( record_property, + props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, test_vector_path, @@ -107,6 +107,7 @@ def test_pca_enc( get_mld_lim, decoder_only, abs_tol, + get_ssnr, ): pca = True tag = tag + fs + "c" @@ -143,6 +144,7 @@ def test_pca_enc( # dec sba_dec( record_property, + props_to_record, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -162,6 +164,7 @@ def test_pca_enc( get_mld_lim=get_mld_lim, pca=pca, abs_tol=abs_tol, + get_ssnr=get_ssnr, ) @@ -174,6 +177,7 @@ def test_pca_enc( @pytest.mark.parametrize("SID", SID_list) def test_sba_enc_system( record_property, + props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, test_vector_path, @@ -194,6 +198,7 @@ def test_sba_enc_system( get_mld_lim, decoder_only, abs_tol, + get_ssnr, ): if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -251,6 +256,7 @@ def test_sba_enc_system( # dec sba_dec( record_property, + props_to_record, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -269,6 +275,7 @@ def test_sba_enc_system( get_mld=get_mld, get_mld_lim=get_mld_lim, abs_tol=abs_tol, + get_ssnr=get_ssnr, ) @@ -277,6 +284,7 @@ def test_sba_enc_system( @pytest.mark.parametrize("tag", tag_list_HOA2) def test_spar_hoa2_enc_system( record_property, + props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, test_vector_path, @@ -292,6 +300,7 @@ def test_spar_hoa2_enc_system( get_mld_lim, decoder_only, abs_tol, + get_ssnr, ): fs = "48" dtx = "0" @@ -326,6 +335,7 @@ def test_spar_hoa2_enc_system( # dec sba_dec( record_property, + props_to_record, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -344,6 +354,7 @@ def test_spar_hoa2_enc_system( get_mld=get_mld, get_mld_lim=get_mld_lim, abs_tol=abs_tol, + get_ssnr=get_ssnr, ) @@ -352,6 +363,7 @@ def test_spar_hoa2_enc_system( @pytest.mark.parametrize("tag", tag_list_HOA3) def test_spar_hoa3_enc_system( record_property, + props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, test_vector_path, @@ -367,6 +379,7 @@ def test_spar_hoa3_enc_system( get_mld_lim, decoder_only, abs_tol, + get_ssnr, ): fs = "48" dtx = "0" @@ -401,6 +414,7 @@ def test_spar_hoa3_enc_system( # dec sba_dec( record_property, + props_to_record, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -419,6 +433,7 @@ def test_spar_hoa3_enc_system( get_mld=get_mld, get_mld_lim=get_mld_lim, abs_tol=abs_tol, + get_ssnr=get_ssnr, ) @@ -429,6 +444,7 @@ def test_spar_hoa3_enc_system( @pytest.mark.parametrize("sample_rate_bw_idx", sample_rate_bw_idx_list) def test_sba_enc_BWforce_system( record_property, + props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, test_vector_path, @@ -446,6 +462,7 @@ def test_sba_enc_BWforce_system( get_mld_lim, decoder_only, abs_tol, + get_ssnr, ): if dtx == "1" and ivas_br not in ["32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -486,6 +503,7 @@ def test_sba_enc_BWforce_system( # dec sba_dec( record_property, + props_to_record, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -504,6 +522,7 @@ def test_sba_enc_BWforce_system( get_mld=get_mld, get_mld_lim=get_mld_lim, abs_tol=abs_tol, + get_ssnr=get_ssnr, ) @@ -646,6 +665,7 @@ def sba_enc( def sba_dec( record_property, + props_to_record, decoder_frontend, ref_decoder_frontend, reference_path, @@ -665,6 +685,7 @@ def sba_dec( get_mld_lim=0, pca=False, abs_tol=0, + get_ssnr=False, ): # -------- run cmd ------------ # sampling rate to BW mapping @@ -729,16 +750,12 @@ def sba_dec( get_mld=get_mld, mld_lim=get_mld_lim, abs_tol=abs_tol, + get_ssnr=get_ssnr, ) - if get_mld: - mld = re.search(MLD_PATTERN, reason).groups(1)[0] - record_property("MLD", mld) - - max_diff = 0 - if cmp_result: - search_result = re.search(MAX_DIFF_PATTERN, reason) - max_diff = search_result.groups(1)[0] - record_property("MAXIMUM ABS DIFF", max_diff) + + props = parse_properties(reason, cmp_result!=0, props_to_record) + for k, v in props.items(): + record_property(k, v) # report compare result if cmp_result != 0: diff --git a/tests/conftest.py b/tests/conftest.py index cec06c2061b767601e6de4f85e13e9f377a330e5..52b32c7704f98ac37dc6a5d65e46321c1798a7fa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,6 +34,7 @@ Pytest customization (configuration and fixtures) for the IVAS codec test suite. import logging import os +import re from tests import testconfig import pytest import platform @@ -41,6 +42,7 @@ import textwrap from pathlib import Path from subprocess import TimeoutExpired, run from typing import Optional, Union +from .constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN logger = logging.getLogger(__name__) USE_LOGGER_FOR_DBG = False # current tests do not make use of the logger feature @@ -171,6 +173,13 @@ def pytest_addoption(parser): help="MLD limit for comparison (default: 0)", default="0", ) + + parser.addoption( + "--ssnr", + action="store_true", + help="Compute Segmental SNR (SSNR) between ref and dut output instead of just comparing for bitexactness", + ) + parser.addoption( "--create_ref", action="store_true", @@ -242,6 +251,14 @@ def get_mld_lim(request): return float(request.config.getoption("--mld-lim")) +@pytest.fixture(scope="session", autouse=True) +def get_ssnr(request): + """ + Return indication to compute ssnr during ref/dut comparison. + """ + return request.config.option.ssnr + + @pytest.fixture(scope="session") def abs_tol(request) -> int: """ @@ -349,7 +366,11 @@ class EncoderFrontend: try: result = run( - command, capture_output=True, check=False, timeout=self.timeout, cwd=run_dir + command, + capture_output=True, + check=False, + timeout=self.timeout, + cwd=run_dir, ) except TimeoutExpired: pytest.fail(f"{self._type} encoder run timed out after {self.timeout}s.") @@ -548,7 +569,11 @@ class DecoderFrontend: try: result = run( - command, capture_output=True, check=False, timeout=self.timeout, cwd=run_dir + command, + capture_output=True, + check=False, + timeout=self.timeout, + cwd=run_dir, ) except TimeoutExpired: pytest.fail(f"{self._type} decoder run timed out after {self.timeout}s.") @@ -749,3 +774,47 @@ def pytest_configure(config): testconfig.MD5_REF_DICT = { line.split()[0]: line.split()[1] for line in f.readlines() } + + +@pytest.fixture(scope="session") +def props_to_record(request, get_mld, get_ssnr) -> str: + props = ["MAXIMUM ABS DIFF"] + if get_mld: + props.append("MLD") + if get_ssnr: + props.append("SSNR") + + return props + + +def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: list): + """ + Record the given properties in the report by parsing their values from the text. + """ + + props = dict() + + for prop in props_to_record: + if prop == "MLD": + mld = float(re.search(MLD_PATTERN, text_to_parse).groups(1)[0]) + props[prop] = mld + elif prop == "MAXIMUM ABS DIFF": + max_diff = 0 + if output_differs: + if (match := re.search(MAX_DIFF_PATTERN, text_to_parse)) is not None: + max_diff = match.groups(1)[0] + else: + raise MaxDiffPatternNotFound() + props[prop] = max_diff + elif prop == "SSNR": + ssnrs = re.findall(SSNR_PATTERN, text_to_parse) + min_ssnr = min(ssnrs) + min_ssnr_channel = ssnrs.index(min_ssnr) + props["MIN_SSNR"] = min_ssnr + props["MIN_SSNR_CHANNEL"] = min_ssnr_channel + + return props + + +class MaxDiffPatternNotFound(Exception): + pass diff --git a/tests/constants.py b/tests/constants.py index 453c07dca83b7edb3dcaeadc859cb164bb6945bc..5bce5ac4e32ce795737ec68fdf166740799a36b6 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -1,3 +1,4 @@ # regex patterns for parsing the output from cmp_pcm -> mainly for BASOP ci MLD_PATTERN = r"MLD: ([\d\.]*)" MAX_DIFF_PATTERN = r"MAXIMUM ABS DIFF: (\d*)" +SSNR_PATTERN = r"Channel \d* SSNR: (nan|[+-]*inf|[\d\.]*)"