diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py
index e3557e252ba32c9b33962f0f7cad677d05d1624d..c849b160be964c75935ffa4e79e858249545e479 100644
--- a/ci/basop-pages/create_report_pages.py
+++ b/ci/basop-pages/create_report_pages.py
@@ -92,11 +92,12 @@ ARROW_DOWN = '⬂'
# expected columns. actual columns are filtered from the incoming data later, this
# is mainly for controlling the order in the output table
-COLUMNS = ["testcase", "Result", "MLD", "MAXIMUM ABS DIFF"]
+COLUMNS = ["testcase", "Result", "MLD", "MAXIMUM ABS DIFF", "MIN_SSNR"]
COLUMNS_GLOBAL = COLUMNS[:1]
COLUMNS_DIFFERENTIAL = COLUMNS[1:]
COLUMNS_DIFFERENTIAL_NOT_MLD = COLUMNS_DIFFERENTIAL[2:]
+
def create_subpage(
html_out,
csv_out,
@@ -111,11 +112,18 @@ def create_subpage(
)
write_out_csv(merged_reports, merged_reports[0].keys(), csv_out)
- table_header_a = "".join([TH_TMPL_GLOBAL.format(c) for c in COLUMNS_GLOBAL] + [TH_TMPL_DIFFERENTIAL.format(c) for c in COLUMNS_DIFFERENTIAL])
+ table_header_a = "".join(
+ [TH_TMPL_GLOBAL.format(c) for c in COLUMNS_GLOBAL]
+ + [TH_TMPL_DIFFERENTIAL.format(c) for c in COLUMNS_DIFFERENTIAL]
+ )
table_header_b = list()
for c in COLUMNS_DIFFERENTIAL:
- table_header_b.append(TH_TMPL_SECOND_ROW.format(f"Previous Run
ID: {id_previous}"))
- table_header_b.append(TH_TMPL_SECOND_ROW.format(f"Current Run
ID: {id_current}"))
+ table_header_b.append(
+ TH_TMPL_SECOND_ROW.format(f"Previous Run
ID: {id_previous}")
+ )
+ table_header_b.append(
+ TH_TMPL_SECOND_ROW.format(f"Current Run
ID: {id_current}")
+ )
table_header_b = "".join(table_header_b)
table_body = "\n".join(
tr_from_row(row, id_current, id_previous) for row in merged_reports
@@ -241,8 +249,13 @@ def merge_and_cleanup_mld_reports(
return diff
- other_col_pairs = [(f"{col}-{id_previous}", f"{col}-{id_current}") for col in COLUMNS_DIFFERENTIAL_NOT_MLD]
- merged = sorted(merged, key=partial(sort_func, other_col_pairs=other_col_pairs), reverse=True)
+ other_col_pairs = [
+ (f"{col}-{id_previous}", f"{col}-{id_current}")
+ for col in COLUMNS_DIFFERENTIAL_NOT_MLD
+ ]
+ merged = sorted(
+ merged, key=partial(sort_func, other_col_pairs=other_col_pairs), reverse=True
+ )
# remove the unecessary whole path from the testcase names
for row in merged:
diff --git a/scripts/parse_mld_xml.py b/scripts/parse_mld_xml.py
index 7370cfe7ff3b39e2bb2319034b06880c77984070..a7f10f94a468968934337438230d0d5d08c2bbe8 100644
--- a/scripts/parse_mld_xml.py
+++ b/scripts/parse_mld_xml.py
@@ -7,7 +7,7 @@ from xml.etree import ElementTree
Parse a junit report and create an MLD summary report.
"""
-PROPERTIES = ["MLD", "MAXIMUM ABS DIFF"]
+PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR"]
# Main routine
diff --git a/scripts/pyaudio3dtools/audioarray.py b/scripts/pyaudio3dtools/audioarray.py
index d89c079d8989d87841c8d95ac2d425639b92d7ca..0c313c4c91da056c46df90bba2a43361d6140da3 100644
--- a/scripts/pyaudio3dtools/audioarray.py
+++ b/scripts/pyaudio3dtools/audioarray.py
@@ -1,36 +1,37 @@
#!/usr/bin/env python3
"""
- (C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB,
- Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
- Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
- Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
- contributors to this repository. All Rights Reserved.
-
- This software is protected by copyright law and by international treaties.
- The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB,
- Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
- Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
- Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
- contributors to this repository retain full ownership rights in their respective contributions in
- the software. This notice grants no license of any kind, including but not limited to patent
- license, nor is any license granted by implication, estoppel or otherwise.
-
- Contributors are required to enter into the IVAS codec Public Collaboration agreement before making
- contributions.
-
- This software is provided "AS IS", without any express or implied warranties. The software is in the
- development stage. It is intended exclusively for experts who have experience with such software and
- solely for the purpose of inspection. All implied warranties of non-infringement, merchantability
- and fitness for a particular purpose are hereby disclaimed and excluded.
-
- Any dispute, controversy or claim arising under or in relation to providing this software shall be
- submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in
- accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and
- the United Nations Convention on Contracts on the International Sales of Goods.
+(C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB,
+Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
+Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
+Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
+contributors to this repository. All Rights Reserved.
+
+This software is protected by copyright law and by international treaties.
+The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB,
+Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
+Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
+Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
+contributors to this repository retain full ownership rights in their respective contributions in
+the software. This notice grants no license of any kind, including but not limited to patent
+license, nor is any license granted by implication, estoppel or otherwise.
+
+Contributors are required to enter into the IVAS codec Public Collaboration agreement before making
+contributions.
+
+This software is provided "AS IS", without any express or implied warranties. The software is in the
+development stage. It is intended exclusively for experts who have experience with such software and
+solely for the purpose of inspection. All implied warranties of non-infringement, merchantability
+and fitness for a particular purpose are hereby disclaimed and excluded.
+
+Any dispute, controversy or claim arising under or in relation to providing this software shall be
+submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in
+accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and
+the United Nations Convention on Contracts on the International Sales of Goods.
"""
import logging
+import warnings
import math
import multiprocessing as mp
import platform
@@ -232,6 +233,10 @@ def compare(
fs: int,
per_frame: bool = True,
get_mld: bool = False,
+ get_ssnr: bool = False,
+ ssnr_thresh_low: float = -np.inf,
+ ssnr_thresh_high: float = np.inf,
+ apply_thresholds_to_ref_only: bool = False,
) -> dict:
"""Compare two audio arrays
@@ -247,6 +252,18 @@ def compare(
Compute difference per frame (default True)
get_mld: bool
Run MLD tool if there is a difference between the signals (default False)
+ get_ssnr: bool
+ Compute Segmental SNR between signals
+ ssnr_thresh_low: float
+ Low threshold for including a segment in the SSNR computation. Per default, both
+ reference and test signal power are compared to this threshold, see below
+ ssnr_thresh_high: float
+ High threshold for including a segment in the SSNR computation. Per default, both
+ reference and test signal power are compared to this threshold, see below
+ apply_thresholds_to_ref_only: bool
+ Set to True to only apply the threshold comparison for the reference signal
+ for whether to include a segment in the ssnr computation. Use this to align
+ behaviour with the MPEG-D conformance specification.
Returns
-------
@@ -266,8 +283,13 @@ def compare(
"first_diff_pos_sample": -1,
"first_diff_pos_channel": -1,
"first_diff_pos_frame": -1,
- "MLD": 0 if get_mld else None,
}
+
+ if get_mld:
+ result["MLD"] = 0
+ if get_ssnr:
+ result["SSNR"] = np.asarray([np.inf] * ref.shape[1])
+
if per_frame:
result["max_abs_diff_pos_frame"] = 0
result["nframes_diff"] = 0
@@ -320,7 +342,6 @@ def compare(
result["nframes_diff_percentage"] = nframes_diff_percentage
if get_mld:
-
mld_max = 0
toolsdir = Path(__file__).parent.parent.joinpath("tools")
if platform.system() == "Windows":
@@ -334,8 +355,14 @@ def compare(
for i in range(nchannels):
tmpfile_ref = Path(tmpdir).joinpath(f"ref_ch{i+1}.wav")
tmpfile_test = Path(tmpdir).joinpath(f"test_ch{i+1}.wav")
- r48 = np.clip( resample(ref[:, i].astype(float), fs, 48000), -32768, 32767 ).astype(np.int16) # Convert to float for resample, then to int16 for wavfile.write
- t48 = np.clip( resample(test[:, i].astype(float), fs, 48000), -32768, 32767 ).astype(np.int16)
+ r48 = np.clip(
+ resample(ref[:, i].astype(float), fs, 48000), -32768, 32767
+ ).astype(
+ np.int16
+ ) # Convert to float for resample, then to int16 for wavfile.write
+ t48 = np.clip(
+ resample(test[:, i].astype(float), fs, 48000), -32768, 32767
+ ).astype(np.int16)
wavfile.write(str(tmpfile_ref), 48000, r48)
wavfile.write(str(tmpfile_test), 48000, t48)
out = subprocess.check_output([mld, tmpfile_ref, tmpfile_test])
@@ -343,6 +370,19 @@ def compare(
result["MLD"] = mld_max
+ if get_ssnr:
+ # length of segment is always 20ms
+ len_seg = int(0.02 * fs)
+ print(len_seg, ref.shape, test.shape)
+ result["SSNR"] = ssnr(
+ ref,
+ test,
+ len_seg,
+ thresh_low=ssnr_thresh_low,
+ thresh_high=ssnr_thresh_high,
+ apply_thresholds_to_ref_only=apply_thresholds_to_ref_only,
+ )
+
return result
@@ -513,3 +553,75 @@ def process_async(files: Iterable, func: Callable, **kwargs):
for r in results:
r.get()
return results
+
+
+def ssnr(
+ ref_sig: np.ndarray,
+ test_sig: np.ndarray,
+ len_seg: int,
+ thresh_low: float = -200,
+ thresh_high: float = 0,
+ apply_thresholds_to_ref_only: bool = False,
+) -> np.ndarray:
+ """
+ Calculate Segmental SNR for test_sig to ref_sig as defined in ISO/IEC 14496-4
+ """
+ ss = list()
+
+ ref_sig_norm = ref_sig / -np.iinfo(np.int16).min
+ test_sig_norm = test_sig / -np.iinfo(np.int16).min
+
+ # check if diff of signal is zero already, then SNR is infinite, since no noise
+ diff_sig_norm = ref_sig_norm - test_sig_norm
+ if np.all(diff_sig_norm == 0):
+ return np.asarray([np.inf] * ref_sig_norm.shape[1])
+
+ channels_identical_idx = np.sum(np.abs(diff_sig_norm), axis=0) == 0
+
+ denom_add = 10**-13 * len_seg
+ segment_counter = np.zeros(ref_sig.shape[1])
+
+ # iterate over test signal too to allow power comparison to threshold
+ for ref_seg, diff_seg, test_seg in zip(
+ get_framewise(ref_sig_norm, len_seg, zero_pad=True),
+ get_framewise(diff_sig_norm, len_seg, zero_pad=True),
+ get_framewise(test_sig_norm, len_seg, zero_pad=True),
+ ):
+ nrg_ref = np.sum(ref_seg**2, axis=0)
+ nrg_diff = np.sum(diff_seg**2, axis=0)
+
+ ss_seg = np.log10(1 + nrg_ref / (denom_add + nrg_diff))
+
+ # only sum up segments that fall inside the thresholds
+ # add small eps to nrg_ref to prevent RuntimeWarnings from numpy
+ ref_power = 10 * np.log10((nrg_ref + 10**-7) / len_seg)
+ zero_mask = np.logical_or(ref_power < thresh_low, ref_power > thresh_high)
+
+ # create same mask for test signal
+ if not apply_thresholds_to_ref_only:
+ nrg_test = np.sum(test_seg**2, axis=0)
+ test_power = 10 * np.log10((nrg_test + 10**-7) / len_seg)
+ zero_mask_test = np.logical_or(
+ test_power < thresh_low, test_power > thresh_high
+ )
+ zero_mask = np.logical_or(zero_mask, zero_mask_test)
+
+ ss_seg[zero_mask] = 0
+ # increase segment counter only for channels that were not zeroed
+ segment_counter += np.logical_not(zero_mask)
+
+ ss.append(ss_seg)
+
+ # if the reference signal was outside the thresholds for all segments in a channel, segment_counter is zero
+ # for that channel and the division here would trigger a warning. We supress the warning and later
+ # set the SSNR for those channels to nan manually instead (overwriting later is simply easier than adding ifs here)
+ with warnings.catch_warnings():
+ ssnr = np.round(
+ 10 * np.log10(10 ** (np.sum(ss, axis=0) / segment_counter) - 1), 2
+ )
+ ssnr[segment_counter == 0] = np.nan
+
+ # this prevents all-zero channels in both signals to be reported as -inf
+ ssnr[channels_identical_idx] = np.inf
+
+ return ssnr
diff --git a/scripts/ssnr.py b/scripts/ssnr.py
new file mode 100644
index 0000000000000000000000000000000000000000..89ac4c6bf999adf8a9530e718a94651e2ba9ac52
--- /dev/null
+++ b/scripts/ssnr.py
@@ -0,0 +1,62 @@
+import argparse
+import sys
+import pathlib
+from pyaudio3dtools import audiofile, audioarray
+
+
+def main(args):
+ ref_sig, fs_ref = audiofile.readfile(args.ref_file)
+ test_sig, fs_test = audiofile.readfile(args.test_file)
+
+ if fs_ref != fs_test:
+ print("Files need to have same sampling rate!")
+ return -1
+
+ len_seg = int(20 * fs_ref / 1000)
+ print(len_seg, ref_sig.shape, test_sig.shape)
+ ssnr = audioarray.ssnr(
+ ref_sig,
+ test_sig,
+ len_seg,
+ args.thresh_low,
+ args.thresh_high,
+ args.apply_thresholds_on_ref_only,
+ )
+
+ for i, s in enumerate(ssnr, start=1):
+ print(f"Channel {i}: {s}")
+
+ return 0
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("ref_file", type=pathlib.Path, help="Reference signal wav file")
+ parser.add_argument(
+ "test_file", type=pathlib.Path, help="Signal under test wav file"
+ )
+ parser.add_argument(
+ "--thresh_low",
+ type=float,
+ default="-inf",
+ help="Low threshold for signal power in a segment to be used in the SSNR calculation (default: -inf).\n"
+ "Applied to both signals per default (see apply_thresholds_on_ref_only argument).",
+ )
+ parser.add_argument(
+ "--thresh_high",
+ type=float,
+ default="inf",
+ help="High threshold for signal power in a segment to be used in the SSNR calculation (default: +inf).\n"
+ "Applied to both signals per default (see apply_thresholds_on_ref_only argument).",
+ )
+ parser.add_argument(
+ "--apply_thresholds_on_ref_only",
+ action="store_true",
+ default=False,
+ help="Use this to apply the thresholding on signal power to the reference signal only.\n"
+ "This makes the implementation behaviour conform to the MPEG-D conformance spec.",
+ )
+
+ args = parser.parse_args()
+
+ sys.exit(main(args))
diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py
index 9783e8b5613cfe89a5ba054b95e3d9cfb5f5c68e..7bf1359f4e9fd7249278638e78138bbc64f008c8 100755
--- a/tests/cmp_pcm.py
+++ b/tests/cmp_pcm.py
@@ -13,14 +13,15 @@ import pyivastest
def cmp_pcm(
- file1,
- file2,
+ ref_file,
+ cmp_file,
out_config,
fs,
get_mld=False,
allow_differing_lengths=False,
mld_lim=0,
abs_tol=0,
+ get_ssnr=False,
) -> (int, str):
"""
Compare 2 PCM files for bitexactness
@@ -39,8 +40,12 @@ def cmp_pcm(
else:
nchannels = pyivastest.constants.OC_TO_NCHANNELS[out_config.upper()]
- s1, _ = pyaudio3dtools.audiofile.readfile(file1, nchannels, fs, outdtype=np.int16)
- s2, _ = pyaudio3dtools.audiofile.readfile(file2, nchannels, fs, outdtype=np.int16)
+ s1, _ = pyaudio3dtools.audiofile.readfile(
+ ref_file, nchannels, fs, outdtype=np.int16
+ )
+ s2, _ = pyaudio3dtools.audiofile.readfile(
+ cmp_file, nchannels, fs, outdtype=np.int16
+ )
# In case of wav input, override the nchannels with the one from the wav header
nchannels = s1.shape[1]
@@ -62,7 +67,13 @@ def cmp_pcm(
return 1, reason
cmp_result = pyaudio3dtools.audioarray.compare(
- s1, s2, fs, per_frame=False, get_mld=get_mld
+ s1,
+ s2,
+ fs,
+ per_frame=False,
+ get_mld=get_mld,
+ get_ssnr=get_ssnr,
+ ssnr_thresh_low=-50,
)
output_differs = 0
@@ -90,13 +101,20 @@ def cmp_pcm(
else:
reason += f" > {mld_lim}"
+ if get_ssnr:
+ reason += "\n"
+ for i, s in enumerate(cmp_result["SSNR"], start=1):
+ msg = f"Channel {i} SSNR: {s}"
+ reason += msg + "\n"
+ print(msg)
+
return output_differs, reason
if __name__ == "__main__":
parser = argparse.ArgumentParser()
- parser.add_argument("file1", type=str)
- parser.add_argument("file2", type=str)
+ parser.add_argument("ref_file", type=str)
+ parser.add_argument("cmp_file", type=str)
parser.add_argument(
"-o",
"--out_config",
diff --git a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py
index 34fa40e278234b784a8c8e9fdf63aac78cdbb3b3..0a2ef6c30b4a1d14dc04e14c6fcdd8cf1c44d31d 100644
--- a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py
+++ b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py
@@ -35,16 +35,13 @@ __doc__ = """
import errno
import os
-import re
from filecmp import cmp
from typing import Optional
import pytest
from tests.cmp_pcm import cmp_pcm
-from tests.conftest import DecoderFrontend, EncoderFrontend
-
-from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN
+from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties
# params
# output_mode_list = ['MONO', 'STEREO', '5_1', '7_1', '5_1_2', '5_1_4', '7_1_4', 'FOA', 'HOA2', 'HOA3', 'BINAURAL', 'BINAURAL_ROOM', 'EXT']
@@ -98,6 +95,7 @@ def check_and_makedir(dir_path):
)
def test_masa_enc_dec(
record_property,
+ props_to_record,
dut_encoder_frontend: EncoderFrontend,
dut_decoder_frontend: DecoderFrontend,
ref_encoder_frontend: EncoderFrontend,
@@ -114,6 +112,7 @@ def test_masa_enc_dec(
get_mld_lim,
decoder_only,
abs_tol,
+ get_ssnr,
):
# Input parameters
in_fs = 48
@@ -206,71 +205,39 @@ def test_masa_enc_dec(
)
# Compare outputs. For EXT output, also compare metadata.
+ metacmp_res = True
if output_mode == "EXT":
# Compare metadata as binary blob
metacmp_res = cmp(dec_met_output_ref, dec_met_output_dut)
- # Compare audio outputs
- pcmcmp_res, reason = cmp_pcm(
- dec_output_dut,
- dec_output_ref,
- output_mode,
- int(out_fs * 1000),
- get_mld=get_mld,
- mld_lim=get_mld_lim,
- abs_tol=abs_tol,
- )
- if get_mld:
- mld = re.search(MLD_PATTERN, reason).groups(1)[0]
- record_property("MLD", mld)
-
- max_diff = 0
- if pcmcmp_res:
- search_result = re.search(MAX_DIFF_PATTERN, reason)
- max_diff = search_result.groups(1)[0]
- record_property("MAXIMUM ABS DIFF", max_diff)
-
- if get_mld and get_mld_lim > 0:
- if pcmcmp_res != 0:
- pytest.fail(reason)
- else:
- # Fail if compare fails compare result
- if metacmp_res == False and pcmcmp_res != 0:
- pytest.fail("Metadata and transport output difference detected")
- elif metacmp_res == False:
- pytest.fail("Metadata output difference detected")
- elif pcmcmp_res != 0:
- pytest.fail("Transport output difference detected")
- else:
- print("Comparison bit exact")
-
- else:
- # Compare audio outputs
- filecmp_res = cmp(dec_output_ref, dec_output_dut)
- if filecmp_res == False:
- cmp_result, reason = cmp_pcm(
- dec_output_dut,
- dec_output_ref,
- output_mode,
- int(out_fs * 1000),
- get_mld=get_mld,
- mld_lim=get_mld_lim,
- abs_tol=abs_tol,
- )
- if get_mld:
- mld = re.search(MLD_PATTERN, reason).groups(1)[0]
- record_property("MLD", mld)
+ # Compare audio outputs
+ pcmcmp_res, reason = cmp_pcm(
+ dec_output_dut,
+ dec_output_ref,
+ output_mode,
+ int(out_fs * 1000),
+ get_mld=get_mld,
+ mld_lim=get_mld_lim,
+ abs_tol=abs_tol,
+ get_ssnr=get_ssnr
+ )
- search_result = re.search(MAX_DIFF_PATTERN, reason)
- max_diff = search_result.groups(1)[0]
- record_property("MAXIMUM ABS DIFF", max_diff)
+ props = parse_properties(reason, pcmcmp_res != 0, props_to_record)
+ for k, v in props.items():
+ record_property(k, v)
- # Report compare result
- if cmp_result != 0:
- pytest.fail(reason)
+ if get_mld and get_mld_lim > 0:
+ if pcmcmp_res != 0:
+ pytest.fail(reason)
+ else:
+ # Fail if compare fails compare result
+ if not metacmp_res and pcmcmp_res != 0:
+ pytest.fail("Metadata and transport output difference detected")
+ elif not metacmp_res:
+ pytest.fail("Metadata output difference detected")
+ elif pcmcmp_res != 0:
+ pytest.fail("Transport output difference detected")
else:
- if get_mld:
- record_property("MLD", "0")
print("Comparison bit exact")
diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py
index eee9fcd9b274eaadbff8e1c21f222ca397a04458..814a6441df7a74d55f8344b10962758ebafc395f 100644
--- a/tests/codec_be_on_mr_nonselection/test_param_file.py
+++ b/tests/codec_be_on_mr_nonselection/test_param_file.py
@@ -35,7 +35,6 @@ Execute tests specified via a parameter file.
import errno
import filecmp
import os
-import re
import platform
from pathlib import Path
from subprocess import run
@@ -43,9 +42,9 @@ import pytest
import numpy as np
from tests.cmp_pcm import cmp_pcm
-from tests.conftest import DecoderFrontend, EncoderFrontend
+from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties
from tests.testconfig import PARAM_FILE
-from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN
+
VALID_DEC_OUTPUT_CONF = [
"MONO",
@@ -136,6 +135,7 @@ def convert_test_string_to_tag(test_string):
@pytest.mark.parametrize("param_file_id", [PARAM_FILE_ID])
def test_param_file_tests(
record_property,
+ props_to_record,
decoder_only,
dut_encoder_frontend: EncoderFrontend,
dut_decoder_frontend: DecoderFrontend,
@@ -152,6 +152,7 @@ def test_param_file_tests(
get_mld,
get_mld_lim,
abs_tol,
+ get_ssnr,
):
enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag]
@@ -341,31 +342,21 @@ def test_param_file_tests(
fs = int(sampling_rate) * 1000
output_differs, reason = cmp_pcm(
- dut_output_file,
ref_output_file,
+ dut_output_file,
output_config,
fs,
get_mld=get_mld,
mld_lim=get_mld_lim,
abs_tol=abs_tol,
allow_differing_lengths=allow_differing_lengths,
+ get_ssnr=get_ssnr,
)
md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config)
- if get_mld:
- mld = re.search(MLD_PATTERN, reason).groups(1)[0]
- record_property("MLD", mld)
-
- max_diff = 0
- if output_differs:
- search_result = re.search(MAX_DIFF_PATTERN, reason)
- if search_result:
- max_diff = search_result.groups(1)[0]
- else:
- msg = "Error " + MAX_DIFF_PATTERN + " not found"
- print(msg)
- pytest.fail(msg)
- record_property("MAXIMUM ABS DIFF", max_diff)
+ props = parse_properties(reason, output_differs, props_to_record)
+ for k, v in props.items():
+ record_property(k, v)
metadata_differs = False
for md_file in md_out_files:
diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py
index 289d10fae131b4997c437ba9eff3e54bc11515ae..e188d4a11454ff08769f4477a2dcec76ff63af59 100644
--- a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py
+++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py
@@ -34,13 +34,12 @@ __doc__ = """
import errno
import os
-import re
import pytest
from tests.cmp_pcm import cmp_pcm
from tests.conftest import DecoderFrontend
-from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN
+from ..conftest import parse_properties
# params
tag_list = ["stvFOA"]
@@ -77,6 +76,7 @@ def check_and_makedir(dir_path):
@pytest.mark.parametrize("gain_flag", gain_list)
def test_sba_plc_system(
record_property,
+ props_to_record,
dut_decoder_frontend: DecoderFrontend,
test_vector_path,
reference_path,
@@ -93,6 +93,7 @@ def test_sba_plc_system(
get_mld,
get_mld_lim,
abs_tol,
+ get_ssnr,
):
SID = 0
if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]:
@@ -115,6 +116,7 @@ def test_sba_plc_system(
# dec
sba_dec_plc(
record_property,
+ props_to_record,
dut_decoder_frontend,
test_vector_path,
reference_path,
@@ -132,6 +134,7 @@ def test_sba_plc_system(
get_mld=get_mld,
get_mld_lim=get_mld_lim,
abs_tol=abs_tol,
+ get_ssnr=get_ssnr,
)
@@ -139,6 +142,7 @@ def test_sba_plc_system(
# -------------------- test function --------------------
def sba_dec_plc(
record_property,
+ props_to_record,
decoder_frontend,
test_vector_path,
reference_path,
@@ -156,6 +160,7 @@ def sba_dec_plc(
get_mld=False,
get_mld_lim=0,
abs_tol=0,
+ get_ssnr=False,
):
# ------------ run cmd ------------
@@ -213,16 +218,12 @@ def sba_dec_plc(
get_mld=get_mld,
mld_lim=get_mld_lim,
abs_tol=abs_tol,
+ get_ssnr=get_ssnr,
)
- if get_mld:
- mld = re.search(MLD_PATTERN, reason).groups(1)[0]
- record_property("MLD", mld)
-
- max_diff = 0
- if cmp_result:
- search_result = re.search(MAX_DIFF_PATTERN, reason)
- max_diff = search_result.groups(1)[0]
- record_property("MAXIMUM ABS DIFF", max_diff)
+
+ props = parse_properties(reason, cmp_result!=0, props_to_record)
+ for k, v in props.items():
+ record_property(k, v)
# report compare result
if cmp_result != 0:
diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py
index 2415d9d9a9cae225bcfc03ae6a274624cf874c74..ce99c55a272c9fa69459d34bea7903e126586965 100644
--- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py
+++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py
@@ -35,14 +35,13 @@ __doc__ = """
import errno
import os
-import re
import pytest
from cut_bs import cut_from_start
from tests.cmp_pcm import cmp_pcm
from tests.conftest import DecoderFrontend, EncoderFrontend
-from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN
+from ..conftest import parse_properties
# params
@@ -92,6 +91,7 @@ def check_and_makedir(dir_path):
@pytest.mark.parametrize("fs", sample_rate_list)
def test_pca_enc(
record_property,
+ props_to_record,
dut_encoder_frontend: EncoderFrontend,
dut_decoder_frontend: DecoderFrontend,
test_vector_path,
@@ -107,6 +107,7 @@ def test_pca_enc(
get_mld_lim,
decoder_only,
abs_tol,
+ get_ssnr,
):
pca = True
tag = tag + fs + "c"
@@ -143,6 +144,7 @@ def test_pca_enc(
# dec
sba_dec(
record_property,
+ props_to_record,
dut_decoder_frontend,
ref_decoder_frontend,
reference_path,
@@ -162,6 +164,7 @@ def test_pca_enc(
get_mld_lim=get_mld_lim,
pca=pca,
abs_tol=abs_tol,
+ get_ssnr=get_ssnr,
)
@@ -174,6 +177,7 @@ def test_pca_enc(
@pytest.mark.parametrize("SID", SID_list)
def test_sba_enc_system(
record_property,
+ props_to_record,
dut_encoder_frontend: EncoderFrontend,
dut_decoder_frontend: DecoderFrontend,
test_vector_path,
@@ -194,6 +198,7 @@ def test_sba_enc_system(
get_mld_lim,
decoder_only,
abs_tol,
+ get_ssnr,
):
if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]:
# skip high bitrates for DTX until DTX issue is resolved
@@ -251,6 +256,7 @@ def test_sba_enc_system(
# dec
sba_dec(
record_property,
+ props_to_record,
dut_decoder_frontend,
ref_decoder_frontend,
reference_path,
@@ -269,6 +275,7 @@ def test_sba_enc_system(
get_mld=get_mld,
get_mld_lim=get_mld_lim,
abs_tol=abs_tol,
+ get_ssnr=get_ssnr,
)
@@ -277,6 +284,7 @@ def test_sba_enc_system(
@pytest.mark.parametrize("tag", tag_list_HOA2)
def test_spar_hoa2_enc_system(
record_property,
+ props_to_record,
dut_encoder_frontend: EncoderFrontend,
dut_decoder_frontend: DecoderFrontend,
test_vector_path,
@@ -292,6 +300,7 @@ def test_spar_hoa2_enc_system(
get_mld_lim,
decoder_only,
abs_tol,
+ get_ssnr,
):
fs = "48"
dtx = "0"
@@ -326,6 +335,7 @@ def test_spar_hoa2_enc_system(
# dec
sba_dec(
record_property,
+ props_to_record,
dut_decoder_frontend,
ref_decoder_frontend,
reference_path,
@@ -344,6 +354,7 @@ def test_spar_hoa2_enc_system(
get_mld=get_mld,
get_mld_lim=get_mld_lim,
abs_tol=abs_tol,
+ get_ssnr=get_ssnr,
)
@@ -352,6 +363,7 @@ def test_spar_hoa2_enc_system(
@pytest.mark.parametrize("tag", tag_list_HOA3)
def test_spar_hoa3_enc_system(
record_property,
+ props_to_record,
dut_encoder_frontend: EncoderFrontend,
dut_decoder_frontend: DecoderFrontend,
test_vector_path,
@@ -367,6 +379,7 @@ def test_spar_hoa3_enc_system(
get_mld_lim,
decoder_only,
abs_tol,
+ get_ssnr,
):
fs = "48"
dtx = "0"
@@ -401,6 +414,7 @@ def test_spar_hoa3_enc_system(
# dec
sba_dec(
record_property,
+ props_to_record,
dut_decoder_frontend,
ref_decoder_frontend,
reference_path,
@@ -419,6 +433,7 @@ def test_spar_hoa3_enc_system(
get_mld=get_mld,
get_mld_lim=get_mld_lim,
abs_tol=abs_tol,
+ get_ssnr=get_ssnr,
)
@@ -429,6 +444,7 @@ def test_spar_hoa3_enc_system(
@pytest.mark.parametrize("sample_rate_bw_idx", sample_rate_bw_idx_list)
def test_sba_enc_BWforce_system(
record_property,
+ props_to_record,
dut_encoder_frontend: EncoderFrontend,
dut_decoder_frontend: DecoderFrontend,
test_vector_path,
@@ -446,6 +462,7 @@ def test_sba_enc_BWforce_system(
get_mld_lim,
decoder_only,
abs_tol,
+ get_ssnr,
):
if dtx == "1" and ivas_br not in ["32000", "64000"]:
# skip high bitrates for DTX until DTX issue is resolved
@@ -486,6 +503,7 @@ def test_sba_enc_BWforce_system(
# dec
sba_dec(
record_property,
+ props_to_record,
dut_decoder_frontend,
ref_decoder_frontend,
reference_path,
@@ -504,6 +522,7 @@ def test_sba_enc_BWforce_system(
get_mld=get_mld,
get_mld_lim=get_mld_lim,
abs_tol=abs_tol,
+ get_ssnr=get_ssnr,
)
@@ -646,6 +665,7 @@ def sba_enc(
def sba_dec(
record_property,
+ props_to_record,
decoder_frontend,
ref_decoder_frontend,
reference_path,
@@ -665,6 +685,7 @@ def sba_dec(
get_mld_lim=0,
pca=False,
abs_tol=0,
+ get_ssnr=False,
):
# -------- run cmd ------------
# sampling rate to BW mapping
@@ -729,16 +750,12 @@ def sba_dec(
get_mld=get_mld,
mld_lim=get_mld_lim,
abs_tol=abs_tol,
+ get_ssnr=get_ssnr,
)
- if get_mld:
- mld = re.search(MLD_PATTERN, reason).groups(1)[0]
- record_property("MLD", mld)
-
- max_diff = 0
- if cmp_result:
- search_result = re.search(MAX_DIFF_PATTERN, reason)
- max_diff = search_result.groups(1)[0]
- record_property("MAXIMUM ABS DIFF", max_diff)
+
+ props = parse_properties(reason, cmp_result!=0, props_to_record)
+ for k, v in props.items():
+ record_property(k, v)
# report compare result
if cmp_result != 0:
diff --git a/tests/conftest.py b/tests/conftest.py
index cec06c2061b767601e6de4f85e13e9f377a330e5..52b32c7704f98ac37dc6a5d65e46321c1798a7fa 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -34,6 +34,7 @@ Pytest customization (configuration and fixtures) for the IVAS codec test suite.
import logging
import os
+import re
from tests import testconfig
import pytest
import platform
@@ -41,6 +42,7 @@ import textwrap
from pathlib import Path
from subprocess import TimeoutExpired, run
from typing import Optional, Union
+from .constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN
logger = logging.getLogger(__name__)
USE_LOGGER_FOR_DBG = False # current tests do not make use of the logger feature
@@ -171,6 +173,13 @@ def pytest_addoption(parser):
help="MLD limit for comparison (default: 0)",
default="0",
)
+
+ parser.addoption(
+ "--ssnr",
+ action="store_true",
+ help="Compute Segmental SNR (SSNR) between ref and dut output instead of just comparing for bitexactness",
+ )
+
parser.addoption(
"--create_ref",
action="store_true",
@@ -242,6 +251,14 @@ def get_mld_lim(request):
return float(request.config.getoption("--mld-lim"))
+@pytest.fixture(scope="session", autouse=True)
+def get_ssnr(request):
+ """
+ Return indication to compute ssnr during ref/dut comparison.
+ """
+ return request.config.option.ssnr
+
+
@pytest.fixture(scope="session")
def abs_tol(request) -> int:
"""
@@ -349,7 +366,11 @@ class EncoderFrontend:
try:
result = run(
- command, capture_output=True, check=False, timeout=self.timeout, cwd=run_dir
+ command,
+ capture_output=True,
+ check=False,
+ timeout=self.timeout,
+ cwd=run_dir,
)
except TimeoutExpired:
pytest.fail(f"{self._type} encoder run timed out after {self.timeout}s.")
@@ -548,7 +569,11 @@ class DecoderFrontend:
try:
result = run(
- command, capture_output=True, check=False, timeout=self.timeout, cwd=run_dir
+ command,
+ capture_output=True,
+ check=False,
+ timeout=self.timeout,
+ cwd=run_dir,
)
except TimeoutExpired:
pytest.fail(f"{self._type} decoder run timed out after {self.timeout}s.")
@@ -749,3 +774,47 @@ def pytest_configure(config):
testconfig.MD5_REF_DICT = {
line.split()[0]: line.split()[1] for line in f.readlines()
}
+
+
+@pytest.fixture(scope="session")
+def props_to_record(request, get_mld, get_ssnr) -> str:
+ props = ["MAXIMUM ABS DIFF"]
+ if get_mld:
+ props.append("MLD")
+ if get_ssnr:
+ props.append("SSNR")
+
+ return props
+
+
+def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: list):
+ """
+ Record the given properties in the report by parsing their values from the text.
+ """
+
+ props = dict()
+
+ for prop in props_to_record:
+ if prop == "MLD":
+ mld = float(re.search(MLD_PATTERN, text_to_parse).groups(1)[0])
+ props[prop] = mld
+ elif prop == "MAXIMUM ABS DIFF":
+ max_diff = 0
+ if output_differs:
+ if (match := re.search(MAX_DIFF_PATTERN, text_to_parse)) is not None:
+ max_diff = match.groups(1)[0]
+ else:
+ raise MaxDiffPatternNotFound()
+ props[prop] = max_diff
+ elif prop == "SSNR":
+ ssnrs = re.findall(SSNR_PATTERN, text_to_parse)
+ min_ssnr = min(ssnrs)
+ min_ssnr_channel = ssnrs.index(min_ssnr)
+ props["MIN_SSNR"] = min_ssnr
+ props["MIN_SSNR_CHANNEL"] = min_ssnr_channel
+
+ return props
+
+
+class MaxDiffPatternNotFound(Exception):
+ pass
diff --git a/tests/constants.py b/tests/constants.py
index 453c07dca83b7edb3dcaeadc859cb164bb6945bc..5bce5ac4e32ce795737ec68fdf166740799a36b6 100644
--- a/tests/constants.py
+++ b/tests/constants.py
@@ -1,3 +1,4 @@
# regex patterns for parsing the output from cmp_pcm -> mainly for BASOP ci
MLD_PATTERN = r"MLD: ([\d\.]*)"
MAX_DIFF_PATTERN = r"MAXIMUM ABS DIFF: (\d*)"
+SSNR_PATTERN = r"Channel \d* SSNR: (nan|[+-]*inf|[\d\.]*)"