From 39dec4c33cb1826470b5df5299df835ae582ef25 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 13 Sep 2022 11:18:45 +0200 Subject: [PATCH 01/14] add blueprint for delay measuring test --- .gitlab-ci.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 624322e87b..50012f3cf1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -342,6 +342,17 @@ self-test-on-merge-request: junit: report-junit.xml +measure-delay: + extends: + - .rules-basis + - .test-job-linux + stage: test + needs: [ "build-codec-linux-cmake" ] + script: + - *print-common-info + + - echo "measure-delay test, yay!" + # --------------------------------------------------------------- # Test jobs for main branch # --------------------------------------------------------------- -- GitLab From aac8f0aa4842f2e173a6a7940f627bc5ab3da3c9 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 14 Sep 2022 11:39:13 +0200 Subject: [PATCH 02/14] add test signal creation script --- ci/generate_delay_measurement_signal.py | 58 +++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 ci/generate_delay_measurement_signal.py diff --git a/ci/generate_delay_measurement_signal.py b/ci/generate_delay_measurement_signal.py new file mode 100644 index 0000000000..612c56e09a --- /dev/null +++ b/ci/generate_delay_measurement_signal.py @@ -0,0 +1,58 @@ +import argparse +import numpy as np + +OC_TO_NCHANNELS = { + "MONO": 1, + "STEREO": 2, + "BINAURAL": 2, + "BINAURAL_ROOM": 2, + "5_1": 6, + "7_1": 8, + "5_1_2": 8, + "5_1_4": 10, + "7_1_4": 12, + "FOA": 4, + "HOA2": 9, + "HOA3": 16, + "EXT": 1, + "ISM1": 1, + "ISM2": 2, + "ISM3": 3, + "ISM4": 4, + "MASA1TC": 1, + "MASA2TC": 2, +} + +SIGNAL_SOURCE = "scripts/testv/stv48c.pcm" +CUT_LEN_SECS = 1 +FS = 48000 + + +def main(args): + channels = OC_TO_NCHANNELS[ args.outformat ] + outfile = args.outfile + + signal = gen_signal(channels) + + # write out interleaved + signal.reshape(-1, 1).tofile(outfile) + + +def gen_signal(channels:int) -> np.ndarray: + # get the measurement signal from the source file (mono file) + signal = np.fromfile(SIGNAL_SOURCE, dtype=np.int16) + # shorten to one second + signal = signal[:FS * CUT_LEN_SECS] + # create other channels with same signal + signal = np.repeat(signal.reshape((-1, 1)), channels, axis=1) + + return signal + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("outformat", type=str, choices=list(OC_TO_NCHANNELS.keys()), help="output format to generate for") + parser.add_argument("outfile", help="output .pcm file with measurement signal") + + args = parser.parse_args() + main(args) \ No newline at end of file -- GitLab From 26cec3c27ccf16208f2331e1fd9887acb5f48768 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 14 Sep 2022 17:37:37 +0200 Subject: [PATCH 03/14] first version of delay test, mono only --- ci/delay_measurement_test.py | 137 +++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 ci/delay_measurement_test.py diff --git a/ci/delay_measurement_test.py b/ci/delay_measurement_test.py new file mode 100644 index 0000000000..152e96c843 --- /dev/null +++ b/ci/delay_measurement_test.py @@ -0,0 +1,137 @@ +# steps: +# 1 define expected delays for all modes/bitrates combination +# 2 generate the test signals +# 3 run all modes with delay compensation +# 4 run all modes without delay compenstaion +# 5 measure delay for all channels and report/compare to expected delay +import subprocess +import os +import json +from tkinter import W +import numpy as np +import sys +import re +PYAUDIO3DTOOLS_PATH = "./scripts/" +sys.path.append(PYAUDIO3DTOOLS_PATH) +from pyaudio3dtools import audioarray, audiofile +import pathlib + +OC_TO_NCHANNELS = { + "MONO": 1, + "STEREO": 2, + "BINAURAL": 2, + "BINAURAL_ROOM": 2, + "5_1": 6, + "7_1": 8, + "5_1_2": 8, + "5_1_4": 10, + "7_1_4": 12, + "FOA": 4, + "HOA2": 9, + "HOA3": 16, + "EXT": 1, + "ISM1": 1, + "ISM2": 2, + "ISM3": 3, + "ISM4": 4, + "MASA1TC": 1, + "MASA2TC": 2, +} + +FORMATS = [ + "MONO", + "STEREO", + *[f"ISM{x}" for x in range(1, 5)], + "5_1", + "5_1_2", + "5_1_4", + "7_1", + "7_1_4", + "SBA", + "MASA1TC", + "MASA2TC" +] + + +TEST_FILE = "./delay_test_file_{}.pcm" +CFG = "ci/delay_test_linux.json" + +def main(): + formats = list(OC_TO_NCHANNELS.keys()) + # for testing + FORMATS = ["mono"] + + form = FORMATS[0] + + # generate test signal + test_file = TEST_FILE.format(form) + gen_cmd = ["python3", "./ci/generate_delay_measurement_signal.py", form, test_file] + subprocess.call(gen_cmd) + + # run format + outfolder_delaycmp = "out_delay_cmp" + run_cmd = [ + "python3", + "./scripts/runIvasCodec.py", + "-C", + form, + "-p", + CFG, + "-I", + test_file, + "-o", + outfolder_delaycmp + ] + subprocess.call(run_cmd) + + outfolder_nodelaycmp = "out_no_delay_cmp" + run_cmd_nodelaycmp = list(run_cmd) + run_cmd_nodelaycmp[-1] = outfolder_nodelaycmp + run_cmd_nodelaycmp.extend([ + "-D=-NO_DELAY_CMP", + "-E=-NO_DELAY_CMP" + ]) + subprocess.call(run_cmd_nodelaycmp) + + get_delay_for_folders(outfolder_delaycmp, outfolder_nodelaycmp) + + +def get_delay_for_folders(folder_delay_cmp: str, folder_no_delay_cmp: str): + + folder_path_delay_cmp = pathlib.Path(folder_delay_cmp) + files_delay_cmp = sorted([ f.name for f in folder_path_delay_cmp.joinpath("dec").iterdir() ]) + + folder_path_no_delay_cmp = pathlib.Path(folder_no_delay_cmp) + files_no_delay_cmp = sorted([ f.name for f in folder_path_no_delay_cmp.joinpath("dec").iterdir() ]) + + assert(files_delay_cmp == files_no_delay_cmp) + + delays_for_samplerates = dict() + for f in files_delay_cmp: + # for testing only + if "dtx" in f or "amr" in f or "_rs" in f: + continue + + f_cmp = folder_path_delay_cmp.joinpath("dec", f) + f_no_cmp = folder_path_no_delay_cmp.joinpath("dec", f) + + s_cmp, fs = audiofile.readfile(f_cmp) + s_no_cmp, fs = audiofile.readfile(f_no_cmp) + + delay = audioarray.getdelay(s_cmp, s_no_cmp) + + match = re.search("b[0-9_]*_[fb|swb|wb|nb]+", f) + bitrate = match.group()[1:-3].replace("_", ".") + + if fs not in delays_for_samplerates: + delays_for_samplerates[fs] = dict() + delays_for_samplerates[fs][bitrate] = delay + + for fs, fs_dict in delays_for_samplerates.items(): + unique_values = np.unique(list(fs_dict.values())) + unique_values = [v / fs for v in unique_values] + print(f"Values for {fs}: {unique_values}") + + +if __name__ == "__main__": + main() \ No newline at end of file -- GitLab From f2d6650fd5bbb7288f35644031eb67a8958fc749 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 14 Sep 2022 17:49:04 +0200 Subject: [PATCH 04/14] filter for modes --- ci/delay_measurement_test.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/ci/delay_measurement_test.py b/ci/delay_measurement_test.py index 152e96c843..e71ac46f3b 100644 --- a/ci/delay_measurement_test.py +++ b/ci/delay_measurement_test.py @@ -39,8 +39,8 @@ OC_TO_NCHANNELS = { } FORMATS = [ - "MONO", - "STEREO", + "mono", + "stereo", *[f"ISM{x}" for x in range(1, 5)], "5_1", "5_1_2", @@ -68,13 +68,25 @@ def main(): gen_cmd = ["python3", "./ci/generate_delay_measurement_signal.py", form, test_file] subprocess.call(gen_cmd) + # collect modes + modes_cmd = [ + "python3", + "./scripts/runIvasCodec.py", + "-l" + ] + modes_proc = subprocess.run(modes_cmd, capture_output=True) + search_str = form + if form in ["5_1", "5_1_2", "5_1_4", "7_1", "7_1_4"]: + search_str = "MC_" + form + modes_list = [m for m in modes_proc.stdout.decode("utf8").split("\n") if form in m] + # run format outfolder_delaycmp = "out_delay_cmp" run_cmd = [ "python3", "./scripts/runIvasCodec.py", - "-C", - form, + "-m", + *modes_list, "-p", CFG, "-I", @@ -96,6 +108,7 @@ def main(): get_delay_for_folders(outfolder_delaycmp, outfolder_nodelaycmp) + def get_delay_for_folders(folder_delay_cmp: str, folder_no_delay_cmp: str): folder_path_delay_cmp = pathlib.Path(folder_delay_cmp) -- GitLab From 01cda9cbfe28e4b4cc2df4d41c629c2aa087954b Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 15 Sep 2022 11:08:29 +0200 Subject: [PATCH 05/14] loop over formats, still buggy though --- ci/delay_measurement_test.py | 199 ++++++++++++++---------- ci/generate_delay_measurement_signal.py | 2 +- 2 files changed, 114 insertions(+), 87 deletions(-) diff --git a/ci/delay_measurement_test.py b/ci/delay_measurement_test.py index e71ac46f3b..08dd68b810 100644 --- a/ci/delay_measurement_test.py +++ b/ci/delay_measurement_test.py @@ -11,6 +11,7 @@ from tkinter import W import numpy as np import sys import re + PYAUDIO3DTOOLS_PATH = "./scripts/" sys.path.append(PYAUDIO3DTOOLS_PATH) from pyaudio3dtools import audioarray, audiofile @@ -39,112 +40,138 @@ OC_TO_NCHANNELS = { } FORMATS = [ - "mono", - "stereo", - *[f"ISM{x}" for x in range(1, 5)], - "5_1", - "5_1_2", - "5_1_4", - "7_1", - "7_1_4", - "SBA", - "MASA1TC", - "MASA2TC" + "mono", + "stereo", + *[f"ISM{x}" for x in range(1, 5)], + "5_1", + "5_1_2", + "5_1_4", + "7_1", + "7_1_4", + "SBA", + "MASA1TC", + "MASA2TC", ] TEST_FILE = "./delay_test_file_{}.pcm" CFG = "ci/delay_test_linux.json" -def main(): - formats = list(OC_TO_NCHANNELS.keys()) - # for testing - FORMATS = ["mono"] - - form = FORMATS[0] - - # generate test signal - test_file = TEST_FILE.format(form) - gen_cmd = ["python3", "./ci/generate_delay_measurement_signal.py", form, test_file] - subprocess.call(gen_cmd) - - # collect modes - modes_cmd = [ - "python3", - "./scripts/runIvasCodec.py", - "-l" - ] - modes_proc = subprocess.run(modes_cmd, capture_output=True) - search_str = form - if form in ["5_1", "5_1_2", "5_1_4", "7_1", "7_1_4"]: - search_str = "MC_" + form - modes_list = [m for m in modes_proc.stdout.decode("utf8").split("\n") if form in m] - - # run format - outfolder_delaycmp = "out_delay_cmp" - run_cmd = [ - "python3", - "./scripts/runIvasCodec.py", - "-m", - *modes_list, - "-p", - CFG, - "-I", - test_file, - "-o", - outfolder_delaycmp - ] - subprocess.call(run_cmd) - - outfolder_nodelaycmp = "out_no_delay_cmp" - run_cmd_nodelaycmp = list(run_cmd) - run_cmd_nodelaycmp[-1] = outfolder_nodelaycmp - run_cmd_nodelaycmp.extend([ - "-D=-NO_DELAY_CMP", - "-E=-NO_DELAY_CMP" - ]) - subprocess.call(run_cmd_nodelaycmp) - - get_delay_for_folders(outfolder_delaycmp, outfolder_nodelaycmp) +def get_modes(format: str) -> list: + modes_cmd = ["python3", "./scripts/runIvasCodec.py", "-l"] + modes_proc = subprocess.run(modes_cmd, capture_output=True) + + search_str = format + # TODO: exclude dtx, rate switching + if format in ["5_1", "5_1_2", "5_1_4", "7_1", "7_1_4"]: + search_str = "MC_" + format + "_b" + modes_list = [ + m + for m in modes_proc.stdout.decode("utf8").split("\n") + if m.startswith(search_str) + and not "_ball_" in m + and not "b_rs" in m + and not "dtx" in m + and not "amr" in m + ] + + return modes_list + + +def main(): + formats = list(OC_TO_NCHANNELS.keys()) + # for testing + + for form in FORMATS[:-2]: + print(form) + + # generate test signal + test_file = TEST_FILE.format(form) + gen_form = form + if form == "SBA": + gen_form = "HOA3" + gen_cmd = [ + "python3", + "./ci/generate_delay_measurement_signal.py", + gen_form, + test_file, + ] + subprocess.call(gen_cmd) + + modes_list = get_modes(form) + print(modes_list) + + # run format + outfolder_delaycmp = "out_delay_cmp" + run_cmd = [ + "python3", + "./scripts/runIvasCodec.py", + "-m", + *modes_list, + "-p", + CFG, + "-I", + test_file, + "-o", + outfolder_delaycmp, + ] + subprocess.call(run_cmd) + + outfolder_nodelaycmp = "out_no_delay_cmp" + run_cmd_nodelaycmp = list(run_cmd) + run_cmd_nodelaycmp[-1] = outfolder_nodelaycmp + run_cmd_nodelaycmp.extend(["-D=-NO_DELAY_CMP", "-E=-NO_DELAY_CMP"]) + subprocess.call(run_cmd_nodelaycmp) + + get_delay_for_folders(outfolder_delaycmp, outfolder_nodelaycmp) def get_delay_for_folders(folder_delay_cmp: str, folder_no_delay_cmp: str): - folder_path_delay_cmp = pathlib.Path(folder_delay_cmp) - files_delay_cmp = sorted([ f.name for f in folder_path_delay_cmp.joinpath("dec").iterdir() ]) + folder_path_delay_cmp = pathlib.Path(folder_delay_cmp) + files_delay_cmp = sorted( + [ + f.name + for f in folder_path_delay_cmp.joinpath("dec").iterdir() + if f.name.endswith(".wav") + ] + ) - folder_path_no_delay_cmp = pathlib.Path(folder_no_delay_cmp) - files_no_delay_cmp = sorted([ f.name for f in folder_path_no_delay_cmp.joinpath("dec").iterdir() ]) + folder_path_no_delay_cmp = pathlib.Path(folder_no_delay_cmp) + files_no_delay_cmp = sorted( + [ + f.name + for f in folder_path_no_delay_cmp.joinpath("dec").iterdir() + if f.name.endswith(".wav") + ] + ) - assert(files_delay_cmp == files_no_delay_cmp) + assert files_delay_cmp == files_no_delay_cmp - delays_for_samplerates = dict() - for f in files_delay_cmp: - # for testing only - if "dtx" in f or "amr" in f or "_rs" in f: - continue + delays_for_samplerates = dict() + for f in files_delay_cmp: - f_cmp = folder_path_delay_cmp.joinpath("dec", f) - f_no_cmp = folder_path_no_delay_cmp.joinpath("dec", f) + f_cmp = folder_path_delay_cmp.joinpath("dec", f) + f_no_cmp = folder_path_no_delay_cmp.joinpath("dec", f) - s_cmp, fs = audiofile.readfile(f_cmp) - s_no_cmp, fs = audiofile.readfile(f_no_cmp) + s_cmp, fs = audiofile.readfile(f_cmp) + s_no_cmp, fs = audiofile.readfile(f_no_cmp) - delay = audioarray.getdelay(s_cmp, s_no_cmp) + delay = audioarray.getdelay(s_cmp, s_no_cmp) - match = re.search("b[0-9_]*_[fb|swb|wb|nb]+", f) - bitrate = match.group()[1:-3].replace("_", ".") + match = re.search("b[0-9_]*_[fb|swb|wb|nb]+", f) + bitrate = match.group()[1:-3].replace("_", ".") - if fs not in delays_for_samplerates: - delays_for_samplerates[fs] = dict() - delays_for_samplerates[fs][bitrate] = delay + if fs not in delays_for_samplerates: + delays_for_samplerates[fs] = dict() + delays_for_samplerates[fs][bitrate] = delay - for fs, fs_dict in delays_for_samplerates.items(): - unique_values = np.unique(list(fs_dict.values())) - unique_values = [v / fs for v in unique_values] - print(f"Values for {fs}: {unique_values}") + for fs, fs_dict in delays_for_samplerates.items(): + unique_values = np.unique(list(fs_dict.values())) + unique_values = [v / fs for v in unique_values] + print(f"Values for {fs}: {unique_values}") if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/ci/generate_delay_measurement_signal.py b/ci/generate_delay_measurement_signal.py index 612c56e09a..4e2fda1968 100644 --- a/ci/generate_delay_measurement_signal.py +++ b/ci/generate_delay_measurement_signal.py @@ -51,7 +51,7 @@ def gen_signal(channels:int) -> np.ndarray: if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("outformat", type=str, choices=list(OC_TO_NCHANNELS.keys()), help="output format to generate for") + parser.add_argument("outformat", type=str.upper, choices=list(OC_TO_NCHANNELS.keys()), help="output format to generate for") parser.add_argument("outfile", help="output .pcm file with measurement signal") args = parser.parse_args() -- GitLab From 674bfd1bed8750426187fe5ad15e17e72ba05e7a Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 15 Sep 2022 11:48:05 +0200 Subject: [PATCH 06/14] change result collection --- ci/delay_measurement_test.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/ci/delay_measurement_test.py b/ci/delay_measurement_test.py index 08dd68b810..126c70e73c 100644 --- a/ci/delay_measurement_test.py +++ b/ci/delay_measurement_test.py @@ -11,6 +11,7 @@ from tkinter import W import numpy as np import sys import re +import pprint PYAUDIO3DTOOLS_PATH = "./scripts/" sys.path.append(PYAUDIO3DTOOLS_PATH) @@ -124,7 +125,10 @@ def main(): run_cmd_nodelaycmp.extend(["-D=-NO_DELAY_CMP", "-E=-NO_DELAY_CMP"]) subprocess.call(run_cmd_nodelaycmp) - get_delay_for_folders(outfolder_delaycmp, outfolder_nodelaycmp) + delays = get_delay_for_folders(outfolder_delaycmp, outfolder_nodelaycmp) + + pp = pprint.PrettyPrinter(indent=4) + pp.pprint(delays) def get_delay_for_folders(folder_delay_cmp: str, folder_no_delay_cmp: str): @@ -149,7 +153,7 @@ def get_delay_for_folders(folder_delay_cmp: str, folder_no_delay_cmp: str): assert files_delay_cmp == files_no_delay_cmp - delays_for_samplerates = dict() + delays = dict() for f in files_delay_cmp: f_cmp = folder_path_delay_cmp.joinpath("dec", f) @@ -158,19 +162,16 @@ def get_delay_for_folders(folder_delay_cmp: str, folder_no_delay_cmp: str): s_cmp, fs = audiofile.readfile(f_cmp) s_no_cmp, fs = audiofile.readfile(f_no_cmp) - delay = audioarray.getdelay(s_cmp, s_no_cmp) + delay = audioarray.getdelay(s_cmp, s_no_cmp) / fs - match = re.search("b[0-9_]*_[fb|swb|wb|nb]+", f) - bitrate = match.group()[1:-3].replace("_", ".") + search_prefix = "delay_test_file_[a-zA-Z]*_" + op_name = re.sub(search_prefix, "", f).split(".wav")[0] - if fs not in delays_for_samplerates: - delays_for_samplerates[fs] = dict() - delays_for_samplerates[fs][bitrate] = delay + if delay not in delays: + delays[delay] = list() + delays[delay].append(op_name) - for fs, fs_dict in delays_for_samplerates.items(): - unique_values = np.unique(list(fs_dict.values())) - unique_values = [v / fs for v in unique_values] - print(f"Values for {fs}: {unique_values}") + return delays if __name__ == "__main__": -- GitLab From 1dc8692851bc2c2dd33f55a172b4a156c378f98d Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 15 Sep 2022 14:38:00 +0200 Subject: [PATCH 07/14] write results to csv file --- ci/delay_measurement_test.py | 57 +++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/ci/delay_measurement_test.py b/ci/delay_measurement_test.py index 126c70e73c..c28b83174e 100644 --- a/ci/delay_measurement_test.py +++ b/ci/delay_measurement_test.py @@ -5,13 +5,9 @@ # 4 run all modes without delay compenstaion # 5 measure delay for all channels and report/compare to expected delay import subprocess -import os -import json -from tkinter import W -import numpy as np import sys import re -import pprint +import pandas as pd PYAUDIO3DTOOLS_PATH = "./scripts/" sys.path.append(PYAUDIO3DTOOLS_PATH) @@ -39,7 +35,6 @@ OC_TO_NCHANNELS = { "MASA1TC": 1, "MASA2TC": 2, } - FORMATS = [ "mono", "stereo", @@ -53,8 +48,6 @@ FORMATS = [ "MASA1TC", "MASA2TC", ] - - TEST_FILE = "./delay_test_file_{}.pcm" CFG = "ci/delay_test_linux.json" @@ -81,11 +74,8 @@ def get_modes(format: str) -> list: def main(): - formats = list(OC_TO_NCHANNELS.keys()) - # for testing - + # TODO: MASA - need metadata for form in FORMATS[:-2]: - print(form) # generate test signal test_file = TEST_FILE.format(form) @@ -101,7 +91,6 @@ def main(): subprocess.call(gen_cmd) modes_list = get_modes(form) - print(modes_list) # run format outfolder_delaycmp = "out_delay_cmp" @@ -126,9 +115,7 @@ def main(): subprocess.call(run_cmd_nodelaycmp) delays = get_delay_for_folders(outfolder_delaycmp, outfolder_nodelaycmp) - - pp = pprint.PrettyPrinter(indent=4) - pp.pprint(delays) + delays.to_csv("delay_test_results.csv") def get_delay_for_folders(folder_delay_cmp: str, folder_no_delay_cmp: str): @@ -153,7 +140,15 @@ def get_delay_for_folders(folder_delay_cmp: str, folder_no_delay_cmp: str): assert files_delay_cmp == files_no_delay_cmp - delays = dict() + output = { + "format": list(), + "fs": list(), + "bitrate": list(), + "bandwidth": list(), + "delay": list(), + "file": list() + } + for f in files_delay_cmp: f_cmp = folder_path_delay_cmp.joinpath("dec", f) @@ -164,14 +159,28 @@ def get_delay_for_folders(folder_delay_cmp: str, folder_no_delay_cmp: str): delay = audioarray.getdelay(s_cmp, s_no_cmp) / fs - search_prefix = "delay_test_file_[a-zA-Z]*_" + search_prefix = "delay_test_file_(([0-9_]+)|[a-zA-Z0-9]+_)" op_name = re.sub(search_prefix, "", f).split(".wav")[0] - - if delay not in delays: - delays[delay] = list() - delays[delay].append(op_name) - - return delays + op_name_split = op_name.split("_b") + + search_br = "b[0-9]+(_[0-9])*" + br = float(re.search(search_br, op_name).group()[1:].replace("_", ".")) + + bw = "fb" + if "_swb_" in op_name: + bw = "swb" + elif "_wb_" in op_name: + bw = "wb" + elif "_nb_" in op_name: + bw = "nb" + + output["format"].append(op_name_split[0]) + output["fs"].append(fs) + output["bitrate"].append(br) + output["bandwidth"].append(bw) + output["delay"].append(delay) + + return pd.DataFrame(output) if __name__ == "__main__": -- GitLab From bcde76058b82e83be658846dfd68e4b7c9430899 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 15 Sep 2022 14:48:30 +0200 Subject: [PATCH 08/14] add printout of results --- ci/delay_measurement_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/delay_measurement_test.py b/ci/delay_measurement_test.py index c28b83174e..d389b7677b 100644 --- a/ci/delay_measurement_test.py +++ b/ci/delay_measurement_test.py @@ -74,7 +74,6 @@ def get_modes(format: str) -> list: def main(): - # TODO: MASA - need metadata for form in FORMATS[:-2]: # generate test signal @@ -115,6 +114,10 @@ def main(): subprocess.call(run_cmd_nodelaycmp) delays = get_delay_for_folders(outfolder_delaycmp, outfolder_nodelaycmp) + + with pd.option_context('display.max_rows', None, 'display.max_columns', None): + print(delays) + delays.to_csv("delay_test_results.csv") -- GitLab From bc78c97ec0d3f0ce605599c7de3c6e8bb5781276 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 15 Sep 2022 14:55:41 +0200 Subject: [PATCH 09/14] trigger delay test on MR pipelines for testing --- .gitlab-ci.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 50012f3cf1..7892729bfa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -345,13 +345,18 @@ self-test-on-merge-request: measure-delay: extends: - .rules-basis - - .test-job-linux + - .rules-merge-request stage: test + tags: + - delay-test needs: [ "build-codec-linux-cmake" ] script: - *print-common-info - - echo "measure-delay test, yay!" + - python3 ci/delay_measurement_test.py + artifacts: + paths: + - delay_test_results.csv # --------------------------------------------------------------- # Test jobs for main branch -- GitLab From 8125e31ebd13a42e325baa8ebdf9fd2ff44cf6bb Mon Sep 17 00:00:00 2001 From: kiene Date: Thu, 15 Sep 2022 13:10:18 +0000 Subject: [PATCH 10/14] create necessary folders --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7892729bfa..5b660dbad7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -353,6 +353,8 @@ measure-delay: script: - *print-common-info + - mkdir out_delay_cmp + - mkdir out_no_delay_cmp - python3 ci/delay_measurement_test.py artifacts: paths: -- GitLab From 62e5fbd2ee83ce35af606b6853f2bb2d2697fb21 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 15 Sep 2022 15:47:57 +0200 Subject: [PATCH 11/14] add config file for delay test script runs --- ci/delay_test_linux.json | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 ci/delay_test_linux.json diff --git a/ci/delay_test_linux.json b/ci/delay_test_linux.json new file mode 100644 index 0000000000..26dfe1ce25 --- /dev/null +++ b/ci/delay_test_linux.json @@ -0,0 +1,25 @@ +{ + "afspPath": "not_needed", + "utilPath": "/tools", + "inpaths": { + "MONO": "delay_test_file_MONO.wav", + "STEREO": "delay_test_file_STEREO.wav", + "FOA": "delay_test_file_FOA.wav", + "HOA2": "delay_test_file_HOA2.wav", + "HOA3": "delay_test_file_HOA3.wav", + "SBA": "delay_test_file_SBA.wav", + "MASA1TC1DIR": "delay_test_file_MASA1TC.wav", + "MASA1TC2DIR": "delay_test_file_MASA1TC.wav", + "MASA2TC1DIR": "delay_test_file_MASA2TC.wav", + "MASA2TC2DIR": "delay_test_file_MASA2TC.wav", + "5_1": "delay_test_file_5_1.wav", + "5_1_2": "delay_test_file_5_1_2.wav", + "5_1_4": "delay_test_file_5_1_4.wav", + "7_1": "delay_test_file_7_1.wav", + "7_1_4": "delay_test_file_7_1_4.wav", + "ISM1": "delay_test_file_ISM1.wav", + "ISM2": "delay_test_file_ISM2.wav", + "ISM3": "delay_test_file_ISM3.wav", + "ISM4": "delay_test_file_ISM4.wav" + } +} -- GitLab From 147bacd092308165588ef6845b4f0718e91f6b89 Mon Sep 17 00:00:00 2001 From: kiene Date: Thu, 15 Sep 2022 14:03:03 +0000 Subject: [PATCH 12/14] add building to test script --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5b660dbad7..5497e5e362 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -353,6 +353,8 @@ measure-delay: script: - *print-common-info + - make -j + - mkdir out_delay_cmp - mkdir out_no_delay_cmp - python3 ci/delay_measurement_test.py -- GitLab From f1010843c76e68e6b052a4ec55b4c40226fd3b74 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 16 Sep 2022 10:52:52 +0200 Subject: [PATCH 13/14] move signal generation to test run script and fix bug --- ci/delay_measurement_test.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/ci/delay_measurement_test.py b/ci/delay_measurement_test.py index d389b7677b..bc3462f61f 100644 --- a/ci/delay_measurement_test.py +++ b/ci/delay_measurement_test.py @@ -7,6 +7,7 @@ import subprocess import sys import re +import numpy as np import pandas as pd PYAUDIO3DTOOLS_PATH = "./scripts/" @@ -48,7 +49,7 @@ FORMATS = [ "MASA1TC", "MASA2TC", ] -TEST_FILE = "./delay_test_file_{}.pcm" +TEST_FILE = "./delay_test_file_{}.wav" CFG = "ci/delay_test_linux.json" @@ -73,6 +74,21 @@ def get_modes(format: str) -> list: return modes_list +SIGNAL_SOURCE = "scripts/testv/stv48c.pcm" +CUT_LEN_SECS = 1 +FS_MEASUREMENT_SIGNAL = 48000 + +def gen_signal(channels:int) -> np.ndarray: + # get the measurement signal from the source file (mono file) + signal = np.fromfile(SIGNAL_SOURCE, dtype=np.int16) + # shorten to one second + signal = signal[:FS_MEASUREMENT_SIGNAL * CUT_LEN_SECS] + # create other channels with same signal + signal = np.repeat(signal.reshape((-1, 1)), channels, axis=1) + + return signal + + def main(): for form in FORMATS[:-2]: @@ -81,13 +97,8 @@ def main(): gen_form = form if form == "SBA": gen_form = "HOA3" - gen_cmd = [ - "python3", - "./ci/generate_delay_measurement_signal.py", - gen_form, - test_file, - ] - subprocess.call(gen_cmd) + sig = gen_signal(OC_TO_NCHANNELS[gen_form.upper()]) + audiofile.writefile(test_file, sig) modes_list = get_modes(form) @@ -149,7 +160,6 @@ def get_delay_for_folders(folder_delay_cmp: str, folder_no_delay_cmp: str): "bitrate": list(), "bandwidth": list(), "delay": list(), - "file": list() } for f in files_delay_cmp: -- GitLab From 3337658717b53656d3b40bd964a2d0a707c4c94a Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 16 Sep 2022 17:21:31 +0200 Subject: [PATCH 14/14] add MASA which was missing --- ci/delay_measurement_test.py | 38 ++++++++++++++++++++++++++++++------ ci/delay_test_linux.json | 8 ++++---- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/ci/delay_measurement_test.py b/ci/delay_measurement_test.py index bc3462f61f..c9245c159f 100644 --- a/ci/delay_measurement_test.py +++ b/ci/delay_measurement_test.py @@ -9,6 +9,7 @@ import sys import re import numpy as np import pandas as pd +import shutil PYAUDIO3DTOOLS_PATH = "./scripts/" sys.path.append(PYAUDIO3DTOOLS_PATH) @@ -46,8 +47,10 @@ FORMATS = [ "7_1", "7_1_4", "SBA", - "MASA1TC", - "MASA2TC", + "MASA_1TC_1DIR", + "MASA_1TC_2DIR", + "MASA_2TC_1DIR", + "MASA_2TC_2DIR", ] TEST_FILE = "./delay_test_file_{}.wav" CFG = "ci/delay_test_linux.json" @@ -90,20 +93,34 @@ def gen_signal(channels:int) -> np.ndarray: def main(): - for form in FORMATS[:-2]: + for form in FORMATS: # generate test signal test_file = TEST_FILE.format(form) gen_form = form if form == "SBA": gen_form = "HOA3" - sig = gen_signal(OC_TO_NCHANNELS[gen_form.upper()]) - audiofile.writefile(test_file, sig) + if "MASA" in form: + # copy over files + masa_source_files = ["scripts/testv/stv_IVASMASA_1dir1TC.pcm"] + for tcs in [1, 2]: + for dir in [1, 2]: + source_signal = f"scripts/testv/stv_IVASMASA_{dir}dir{tcs}TC.pcm" + source_metadata = f"scripts/testv/stv_IVASMASA_{dir}dir{tcs}TC.met" + target_signal = f"delay_test_file_MASA_{tcs}TC_{dir}DIR.wav" + target_metadata = f"delay_test_file_MASA_{tcs}TC_{dir}DIR.met" + s, fs = audiofile.readfile(source_signal, nchannels=tcs) + audiofile.writefile(target_signal, s) + shutil.copy(source_metadata, target_metadata) + else: + sig = gen_signal(OC_TO_NCHANNELS[gen_form.upper()]) + audiofile.writefile(test_file, sig) modes_list = get_modes(form) # run format outfolder_delaycmp = "out_delay_cmp" + run_cmd = [ "python3", "./scripts/runIvasCodec.py", @@ -156,6 +173,7 @@ def get_delay_for_folders(folder_delay_cmp: str, folder_no_delay_cmp: str): output = { "format": list(), + "out_format": list(), "fs": list(), "bitrate": list(), "bandwidth": list(), @@ -174,6 +192,9 @@ def get_delay_for_folders(folder_delay_cmp: str, folder_no_delay_cmp: str): search_prefix = "delay_test_file_(([0-9_]+)|[a-zA-Z0-9]+_)" op_name = re.sub(search_prefix, "", f).split(".wav")[0] + + out_format = op_name.split(".dec.")[-1] + op_name_split = op_name.split("_b") search_br = "b[0-9]+(_[0-9])*" @@ -187,7 +208,12 @@ def get_delay_for_folders(folder_delay_cmp: str, folder_no_delay_cmp: str): elif "_nb_" in op_name: bw = "nb" - output["format"].append(op_name_split[0]) + form = op_name_split[0] + if "MASA" in form: + form = form[form.index("MASA"):] + + output["format"].append(form) + output["out_format"].append(out_format) output["fs"].append(fs) output["bitrate"].append(br) output["bandwidth"].append(bw) diff --git a/ci/delay_test_linux.json b/ci/delay_test_linux.json index 26dfe1ce25..c71ddd543a 100644 --- a/ci/delay_test_linux.json +++ b/ci/delay_test_linux.json @@ -8,10 +8,10 @@ "HOA2": "delay_test_file_HOA2.wav", "HOA3": "delay_test_file_HOA3.wav", "SBA": "delay_test_file_SBA.wav", - "MASA1TC1DIR": "delay_test_file_MASA1TC.wav", - "MASA1TC2DIR": "delay_test_file_MASA1TC.wav", - "MASA2TC1DIR": "delay_test_file_MASA2TC.wav", - "MASA2TC2DIR": "delay_test_file_MASA2TC.wav", + "MASA1TC1DIR": "delay_test_file_MASA_1TC_1DIR.wav", + "MASA1TC2DIR": "delay_test_file_MASA_1TC_2DIR.wav", + "MASA2TC1DIR": "delay_test_file_MASA_2TC_1DIR.wav", + "MASA2TC2DIR": "delay_test_file_MASA_2TC_2DIR.wav", "5_1": "delay_test_file_5_1.wav", "5_1_2": "delay_test_file_5_1_2.wav", "5_1_4": "delay_test_file_5_1_4.wav", -- GitLab