diff --git a/scripts/ivas_conformance/README.md b/scripts/ivas_conformance/README.md index e5fb0824ff9854e03a17ed52893398e94dc13e68..0f617246b1aeb1446756c22500b8f87431b40cb5 100644 --- a/scripts/ivas_conformance/README.md +++ b/scripts/ivas_conformance/README.md @@ -1,22 +1,316 @@ -# IVAS conformance scripts +# IVAS Conformance Scripts -This folder contains scripts for running IVAS conformance tests. This is a placeholder file for instructions. +This folder contains scripts for running IVAS conformance tests. -no-BE conformance USAGE +## Setup for Reference Platform -Following CMDs needs to be executed from ivas-codec root folder: + Reference platform is Ubuntu 24.04 -################generate testvec package and encoder refs (temporary step to obtain testvec package, this step will be removed in final delivery) ############ -sh ivas_be_conf_test_gen.sh -PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/ref_bin --cut_build_path=testvec/cut_bin --test-mode=ENC --regenerate-enc-refs +- Verify the Ubuntu Linux release is 24.04 + ```shell + lsb_release -d | grep Ubuntu + ``` + # It might be similar to Ubuntu 24.04.3 LTS -Encoder conformance: - - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/ref_bin --cut_build_path=testvec/cut_bin --test-mode=ENC -Decoder conformance: - - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/ref_bin --cut_build_path=testvec/cut_bin --test-mode=DEC -Renderer conformance: - - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/ref_bin --cut_build_path=testvec/cut_bin --test-mode=REND -Split renderer confomance: - - +- Install Clang 18 compiler + + ```shell + sudo apt install clang-18 + clang-18 --version + ``` + + Example version observed on Ubuntu 24.04.3 LTS + + ```text + Ubuntu clang version 18.1.3 (1ubuntu1) + Target: x86_64-pc-linux-gnu + Thread model: posix + InstalledDir: /usr/bin + ``` + + It might be required to set Clang-18 as the default clang on the machine + + ```shell + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 100 + ``` + +- Install Python3.13 on Ubuntu 24.04 LTS + + ```shell + sudo apt install python3.13 python3.13-venv + ``` + +- Create virtual environment for Python 3.13 and install requirements + + ```shell + python3.13 -m venv pyConformance + source pyConformance/bin/activate + cd ivas-codec + python -m pip install -r tests/requirements.txt + ``` + +## Reference Conformance Package Generation + +
+ Expand for detailed procedure + +To generate reference conformance package for distribution + +### Generate Reference Outputs and Readme.txt files + + ```shell + sh scripts/ivas_conformance/ivas_be_conf_test_gen.sh + ``` + +
+ Example output of reference test generation +

+::::::::::::::::::::::::
+------------------------------------------
+Generated html report: file:///home/dolby/git/ivas-codec/report_cmd.htm
+------------------------------------------
+=================================================
+2571 passed, 538 skipped, 230 xfailed in 377.10s (0:06:17)
+=================================================
+Identified 5430 files from scripts
+Removed 1515 files
+Kept 5422 files
+
+
+ +### Generate Reference Decoded Outputs for the Reference Encoded files + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --regenerate-enc-refs + ``` + +### Generate a conformance package zip + + ```shell + zip -r conformance.zip testvec scripts/ivas_conformance scripts/tools + ``` + +
+ +## Run CUT tests on Target platform + + To run CUT binaries on the targeted platform, it is necessary to replicate the initial setup for python and dependency packages. The CUT build of the IVAS binaries should be made available in a selected folder and needed for the next step + + To run IVAS DUT commands on the TARGET platform (may be different from ubuntu/linux) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR + ``` + +
+Example Output of CUT execution +

+Accumulating commands from Readme_IVAS_dec.txt
+Accumulating commands from Readme_IVAS_rend.txt
+Accumulating commands from Readme_IVAS_enc.txt
+Accumulating commands from Readme_IVAS_ISAR_post_rend.txt
+Accumulating commands from Readme_IVAS_ISAR_dec.txt
+Accumulating commands from Readme_IVAS_JBM_dec.txt
+No of tests :
+    ENC : 381
+    DEC : 637
+    REND : 666
+    ISAR_ENC : 1032
+    ISAR : 1032
+Executing tests for ENC   (381 tests)
+Executing tests for DEC   (637 tests)
+Executing tests for REND   (666 tests)
+Executing tests for ISAR_ENC   (1032 tests)
+Executing tests for ISAR   (1032 tests)
+
+
+ + This should generate outputs in scripts/CUT_OUTPUTS folder which looks like below:- + + ```shell + CUT_OUTPUTS + +- runlog.txt : Dump of all the commands run and the outputs (mostly jumbled up due to multiprocessing) + +- failedCmds.txt : Log of all the shell commands that failed execution + +- dec/ : Folder containing all decoder tests CUT outputs + +- enc/ : Folder containing all encoder tests CUT outputs + +- renderer_short/ : Folder containing all renderer tests CUT outputs + +- split_rendering/ : Folder containing all split rendering enc/dec tests + ``` + +## Perform the MLD based analysis on the CUT outputs on reference platform (Ubuntu 24.04) + + If CUT test execution is done on a different platform, the scripts/CUT_OUTPUTS must be copied and provided in the reference platform's scripts/CUT_OUTPUTS. Then the following command is used to perform MLD based analysis on the same, encoded outputs will be implicitly decoded using reference decoder executables and MLD analysis performed on the reference decoded outputs. + + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse + ``` + +
+Example Output of CUT Analysis +

+Accumulating commands from Readme_IVAS_dec.txt
+Accumulating commands from Readme_IVAS_rend.txt
+Accumulating commands from Readme_IVAS_enc.txt
+Accumulating commands from Readme_IVAS_ISAR_post_rend.txt
+Accumulating commands from Readme_IVAS_ISAR_dec.txt
+Accumulating commands from Readme_IVAS_JBM_dec.txt
+No of tests :
+    ENC : 381
+    DEC : 637
+    REND : 666
+    ISAR_ENC : 1032
+    ISAR : 1032
+Analysing tests for ENC   (381 tests)
+##########################################################
+<ENC> Total Frames: 2635800
+<ENC> MAX MLD across all frames : 0.0
+<ENC> Frames with MLD == 0 : 2635800 frames (100.0%)
+<ENC> Frames with MLD <= 1 : 2635800 frames (100.0%)
+<ENC> Frames with MLD <= 2 : 2635800 frames (100.0%)
+<ENC> Frames with MLD <= 5 : 2635800 frames (100.0%)
+<ENC> BE frames percentage = 100.0
+<ENC> max absolute diff = 0.0, sample range (-32768, 32767)
+##########################################################
+Analysing tests for DEC   (637 tests)
+##########################################################
+<DEC> Total Frames: 4342140
+<DEC> MAX MLD across all frames : 0.0
+<DEC> Frames with MLD == 0 : 4342140 frames (100.0%)
+<DEC> Frames with MLD <= 1 : 4342140 frames (100.0%)
+<DEC> Frames with MLD <= 2 : 4342140 frames (100.0%)
+<DEC> Frames with MLD <= 5 : 4342140 frames (100.0%)
+<DEC> BE frames percentage = 100.0
+<DEC> max absolute diff = 0.0, sample range (-32768, 32767)
+##########################################################
+Analysing tests for REND   (666 tests)
+##########################################################
+<REND> Total Frames: 4799952
+<REND> MAX MLD across all frames : 0.0
+<REND> Frames with MLD == 0 : 4799952 frames (100.0%)
+<REND> Frames with MLD <= 1 : 4799952 frames (100.0%)
+<REND> Frames with MLD <= 2 : 4799952 frames (100.0%)
+<REND> Frames with MLD <= 5 : 4799952 frames (100.0%)
+<REND> BE frames percentage = 100.0
+<REND> max absolute diff = 0.0, sample range (-32768, 32767)
+##########################################################
+Analysing tests for ISAR_ENC   (1032 tests)
+##########################################################
+<ISAR_ENC> Total Frames: 2125956
+<ISAR_ENC> MAX MLD across all frames : 0.0
+<ISAR_ENC> Frames with MLD == 0 : 2125956 frames (100.0%)
+<ISAR_ENC> Frames with MLD <= 1 : 2125956 frames (100.0%)
+<ISAR_ENC> Frames with MLD <= 2 : 2125956 frames (100.0%)
+<ISAR_ENC> Frames with MLD <= 5 : 2125956 frames (100.0%)
+<ISAR_ENC> BE frames percentage = 100.0
+<ISAR_ENC> max absolute diff = 0.0, sample range (-32768,32767)
+##########################################################
+Analysing tests for ISAR   (1032 tests)
+##########################################################
+<ISAR> Total Frames: 2125956
+<ISAR> MAX MLD across all frames : 0.0
+<ISAR> Frames with MLD == 0 : 2125956 frames (100.0%)
+<ISAR> Frames with MLD <= 1 : 2125956 frames (100.0%)
+<ISAR> Frames with MLD <= 2 : 2125956 frames (100.0%)
+<ISAR> Frames with MLD <= 5 : 2125956 frames (100.0%)
+<ISAR> BE frames percentage = 100.0
+<ISAR> max absolute diff = 0.0, sample range (-32768, 32767)
+##########################################################
+
+
+ +## Executing specific tests only + +All CUT tests can be run specifically for IVAS Encoder,IVAS Decoder,IVAS Renderer, ISAR Encoder and ISAR Decoder only. The commandline allows for ```-test-mode=``` for this functionality, examples : - + +- Run DUT IVAS Encoder Tests Only (on Target Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ENC + ``` + +- Analyse BE conformance for DUT IVAS Encoder Outputs Only (on Reference Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ENC --analyse --be-test + ``` + +- Analyse NON-BE conformance for DUT IVAS Encoder Outputs Only (on Reference Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ENC --analyse + ``` + +- Run DUT IVAS Decoder Tests Only (on Target Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=DEC + ``` + +- Analyse BE conformance for DUT IVAS Decoder Outputs Only + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=DEC --analyse --be-test + ``` + +- Analyse NON-BE conformance DUT IVAS Decoder Outputs Only (on Reference Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=DEC --analyse + ``` + +- Run DUT IVAS Renderer Tests Only (on Target Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=REND + ``` + +- Analyse BE conformance for DUT Renderer Outputs Only + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=REND --analyse --be-test + ``` + +- Analyse NON-BE conformance DUT Renderer Outputs Only + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=REND --analyse + ``` + +- Run DUT ISAR Encoder Tests Only (on Target Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ISAR_ENC + ``` + +- Analyse BE conformance for DUT ISAR Encoder Outputs Only (on Reference Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ISAR_ENC --analyse --be-test + ``` + +- Analyse NON-BE conformance for DUT ISAR Encoder Outputs Only (on Reference Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ISAR_ENC --analyse + ``` + +- Run DUT ISAR Decoder Tests Only (on Target Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ISAR + ``` + +- Analyse BE conformance for DUT ISAR Decoder Outputs Only + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=ISAR --analyse --be-test + ``` + +- Analyse NON-BE conformance DUT ISAR Decoder Outputs Only + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=ISAR --analyse + ``` diff --git a/ivas_be_conf_test_gen.sh b/scripts/ivas_conformance/ivas_be_conf_test_gen.sh similarity index 97% rename from ivas_be_conf_test_gen.sh rename to scripts/ivas_conformance/ivas_be_conf_test_gen.sh index db23621c4a7060e471fde05dc10531b34ea443d1..aa80ed533dc52ebc4058313f293f4d6c86407a44 100644 --- a/ivas_be_conf_test_gen.sh +++ b/scripts/ivas_conformance/ivas_be_conf_test_gen.sh @@ -8,7 +8,6 @@ cp IVAS_cod IVAS_cod_ref cp IVAS_dec IVAS_dec_ref cp IVAS_rend IVAS_rend_ref cp ISAR_post_rend ISAR_post_rend_ref -python3 scripts/prepare_combined_format_inputs.py python3 -m pytest -q tests/codec_be_on_mr_nonselection tests/renderer_short/test_renderer.py tests/split_rendering/test_split_rendering.py -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt rm -rf testvec diff --git a/scripts/ivas_conformance/requirements.txt b/scripts/ivas_conformance/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7f20a7e547b6297d00ee41086e8b202287debece --- /dev/null +++ b/scripts/ivas_conformance/requirements.txt @@ -0,0 +1,3 @@ +scipy>=1.5.2 +numpy>=1.19.2 + diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 15f9fbcd534704c4202dd7660a7deb3010488883..bb56bfacc6fb4a9bd6509c037f9c0fdda5151a8d 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -37,39 +37,201 @@ import numpy as np import subprocess import tempfile import sys -from typing import Optional -from multiprocessing import Process, Value +from typing import Tuple +from multiprocessing import Pool +from dataclasses import dataclass +from typing import Union import shutil +import scipy.io.wavfile as wav +import warnings +import math +import scipy.signal as sig +import filecmp sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) -from pyaudio3dtools.audiofile import readfile, writefile -from pyaudio3dtools.audioarray import resample +def readfile( + filename: str, nchannels: int = 1, fs: int = 48000, outdtype="float" +) -> Tuple[np.ndarray, int]: + """Read audio file (.pcm or .wav) + + Parameters + ---------- + filename: str + Input file path + nchannels: Optional[int] + Number of input channels, required for .pcm otherwise default = 1 + fs: Optional[int] + Input sampling rate, required for .pcm input file, otherwise default = 48000 (Hz) + outdtype: Optional[int] + Data type of output array, python builtin or np.dtype + + Returns + ------- + x: np array + audio signal array + fs: int + signal sampling frequency + + """ + _, file_extension = os.path.splitext(os.path.basename(filename)) + + if file_extension == ".wav": + fs, data = wav.read(filename) + if data.dtype == np.int32: + data = np.interp( + data, + (np.iinfo(np.int32).min, np.iinfo(np.int32).max), + (np.iinfo(np.int16).min, np.iinfo(np.int16).max), + ) + elif data.dtype == np.float32: + data = np.interp( + data, + (-1, 1), + (np.iinfo(np.int16).min, np.iinfo(np.int16).max), + ) + x = np.array(data, dtype=outdtype) + file_len = x.shape[0] + if x.ndim == 1: + # force to be a mtx + x = np.reshape(x, (file_len, 1)) + elif file_extension == ".pcm" or file_extension == ".raw": + x = np.fromfile(filename, dtype=np.int16).astype(outdtype) + signal_len = len(x) // nchannels + x = x.reshape(signal_len, nchannels) + else: + raise ValueError("Wrong input format. Use wav or pcm") + + return x, fs + + +def writefile(filename: str, x: np.ndarray, fs: int = 48000) -> None: + """Write audio file (.pcm or .wav) + + Parameters + ---------- + filename: str + Output file path (.pcm or .wav) + x: np array + Numpy 2D array of dimension: number of samples x number of channels + fs: Optional[int] + Output sampling rate, required for .pcm input file, otherwise default = 48000 (Hz) + + Returns + ------- + None + + """ + _, file_extension = os.path.splitext(os.path.basename(filename)) + + clipped_samples = np.sum( + np.logical_or(x < np.iinfo(np.int16).min, x > np.iinfo(np.int16).max) + ) + if clipped_samples > 0: + warnings.warn(f" Warning: {clipped_samples} samples clipped") + x = np.clip(x, np.iinfo(np.int16).min, np.iinfo(np.int16).max) + + if file_extension == ".wav": + x = x.astype(np.int16) + wav.write(filename, fs, x) + elif file_extension == ".pcm" or file_extension == ".raw": + x = x.astype("int16").reshape(-1, 1) + x.tofile(filename) + else: + raise ValueError("Wrong input format. Use wav or pcm") + + +def resample(x: np.ndarray, in_freq: int, out_freq: int) -> np.ndarray: + """Resample a multi-channel audio array + + Parameters + ---------- + x: numpy array + Input array + in_fs: int + Input sampling rate + out_fs: int + Output sampling rate + + Returns + ------- + y: + Output resampled numpy array + + """ + + if in_freq == out_freq or out_freq is None: + y = x + else: + # get gcd of original and deisred frequency + gcd = math.gcd(in_freq, out_freq) + + # calculate up-sampling factor + up_factor = int(out_freq / gcd) + + # calculate downsampling factor + down_factor = int(in_freq / gcd) + + # resample data using polyphase filtering across columns/channels + if x.ndim == 2: + y = sig.resample_poly(x[:, 0], up_factor, down_factor) + y = np.reshape(y, (y.shape[0], 1)) + for k in range(1, x.shape[1]): + a = sig.resample_poly(x[:, k], up_factor, down_factor) + a = np.reshape(a, (a.shape[0], 1)) + y = np.append(y, a, axis=1) + else: + y = sig.resample_poly(x, up_factor, down_factor) -class MLDConformance: - IVAS_Bins = { - "ENC": "IVAS_cod", - "DEC": "IVAS_dec", - "REND": "IVAS_rend", - "ISAR": "ISAR_post_rend", - } + return y + + +IVAS_Bins = { + "ENC": "IVAS_cod", + "DEC": "IVAS_dec", + "REND": "IVAS_rend", + "ISAR_ENC": "IVAS_dec", + "ISAR": "ISAR_post_rend", +} + + +@dataclass +class TestDesciptor: + dutOutput: str = "" + refOutput: str = "" + rawCmdline: str = "" + dutCmdline: str = "" + refCmdline: str = "" + +@dataclass +class BitstrmTestDescriptor(TestDesciptor): + rawDecCmdline: str = "" + dutDecCmdline: str = "" + refDecCmdline: str = "" + + +class MLDConformance: def setupCommon(self): self.Commands = dict() + self.TestDesc = dict[ + str, dict[str, Union[TestDesciptor, BitstrmTestDescriptor]] + ]() self.EncoderToDecoderCmdMap = dict() - for tag in MLDConformance.IVAS_Bins.keys(): + self.IsarEncoderToDecoderCmdMap = dict() + for tag in IVAS_Bins.keys(): self.Commands[tag] = list() # CREATE OUTPUT DIRECTORY STRUCTURE : CLEAN PREV OUTPUT self.outputDir = os.path.join(self.scriptsDir, "CUT_OUTPUTS") - if os.path.exists(self.outputDir): + if self.args.clean_output_dir and os.path.exists(self.outputDir): shutil.rmtree(self.outputDir, ignore_errors=False) os.makedirs(self.outputDir, exist_ok=True) subdirs = ["enc", "dec", "renderer_short", "split_rendering"] for odir in subdirs: - os.makedirs(os.path.join(self.outputDir, "ref", odir), exist_ok=True) - os.makedirs(os.path.join(self.outputDir, "dut", odir), exist_ok=True) + os.makedirs(os.path.join(self.testvDir, odir), exist_ok=True) + os.makedirs(os.path.join(self.outputDir, odir), exist_ok=True) self.logFile = os.path.join(self.outputDir, "runlog.txt") self.failedCmdsFile = os.path.join(self.outputDir, "failedCmds.txt") @@ -79,16 +241,19 @@ class MLDConformance: def setupDUT(self): self.cut_build_path = args.cut_build_path self.filter = args.filter - self.mldbin = os.path.join(self.toolsdir, platform.system(), "mld") + exe_platform = platform.system() + if exe_platform == "Windows": + exe_platform = "Win32" + self.wavdiffbin = os.path.join(self.toolsdir, exe_platform, "wav-diff") self.CutBins = dict() self.mldcsv = dict() + self.BEcsv = dict() self.sampleStats = dict() - for tag in MLDConformance.IVAS_Bins.keys(): - self.CutBins[tag] = os.path.join( - self.cut_build_path, MLDConformance.IVAS_Bins[tag] - ) + for tag in IVAS_Bins.keys(): + self.CutBins[tag] = os.path.join(self.cut_build_path, IVAS_Bins[tag]) self.mldcsv[tag] = os.path.join(self.outputDir, f"mld_{tag}.csv") + self.BEcsv[tag] = os.path.join(self.outputDir, f"BE_{tag}.csv") self.sampleStats[tag] = os.path.join( self.outputDir, f"sampleStats_{tag}.csv" ) @@ -96,16 +261,13 @@ class MLDConformance: def setupRef(self): self.RefBins = dict() self.ref_build_path = self.args.ref_build_path - for tag in MLDConformance.IVAS_Bins.keys(): - self.RefBins[tag] = os.path.join( - self.ref_build_path, MLDConformance.IVAS_Bins[tag] - ) + for tag in IVAS_Bins.keys(): + self.RefBins[tag] = os.path.join(self.ref_build_path, IVAS_Bins[tag]) def setup(self): self.setupCommon() self.setupRef() - if not self.args.regenerate_enc_refs: - self.setupDUT() + self.setupDUT() def __init__(self, args) -> None: self.args = args @@ -113,35 +275,31 @@ class MLDConformance: self.testvecDir = args.testvecDir self.toolsdir = os.path.join(self.scriptsDir, "tools") self.testvDir = os.path.join(self.testvecDir, "testv") - self.executedTests = Value("i", 0) - self.failedTests = Value("i", 0) self.setup() def accumulateCommands(self): for root, _, files in os.walk(self.testvecDir): for file_name in files: basename, ext = os.path.splitext(file_name) - if ( - ("Readme_IVAS_" in basename) - and ext == ".txt" - and not ("ISAR" in basename) - ): + if ("Readme_IVAS_" in basename) and ext == ".txt": print(f"Accumulating commands from {file_name}") + isISAREnc = "IVAS_ISAR_dec" in file_name file = os.path.join(root, file_name) - self.parseCommandsFile(file) - self.mapEncoderToDecoderCommands() - print("No of tests :") - for key in self.Commands.keys(): - print(f" {key} : {len(self.Commands[key])}") - - def parseCommandsFile(self, filePath): - with open(filePath) as fp: - for line in fp.readlines(): - m = re.search(r"^\$(CUT_.+_BIN) ", line) - if m: - tag = m.group(1).split("_")[1] - if tag in self.Commands.keys(): - self.Commands[tag].append(line) + with open(file) as fp: + for line in fp.readlines(): + m = re.search(r"^\$(CUT_.+_BIN) ", line) + if m: + tag = ( + m.group(1).split("_")[1] + if not isISAREnc + else "ISAR_ENC" + ) + self.Commands[tag].append(line) + self.TestDesc = self.createTestDescriptors() + + def getRendPyTestTag(self, command: str) -> str: + refRendOutputFile = self.getRendOutputFile(command) + return os.path.basename(refRendOutputFile).split(".")[-2] def getPcmPytestTag(self, command: str) -> str: decInput = ( @@ -151,143 +309,259 @@ class MLDConformance: ) return decInput.split(".")[-2] + def getIsarDecPytestTag(self, command: str) -> str: + getName = False + for command in command.split(): + if getName: + return os.path.basename(command).split(".")[-3] + getName = True if command == "-i" else getName + assert False, f"No match found for {command}" + def getEncPytestTag(self, command: str) -> str: return os.path.basename(command.split()[-1]).split(".")[-2] - def mapEncoderToDecoderCommands(self): - decoderPyTestTags = dict() - encoderPyTestTags = dict() - for idx, command in enumerate(self.Commands["DEC"]): - decoderPyTestTags[self.getPcmPytestTag(command)] = idx - for idx, command in enumerate(self.Commands["ENC"]): - encoderPyTestTags[self.getEncPytestTag(command)] = idx - - for encTag in encoderPyTestTags.keys(): - if encTag in decoderPyTestTags.keys(): - self.EncoderToDecoderCmdMap[encoderPyTestTags[encTag]] = ( - decoderPyTestTags[encTag] + def getIsarEncPytestTag(self, command: str) -> str: + return os.path.basename(command.split()[-1]).split(".")[-3] + + def createTestDescriptors( + self, + ) -> dict[str, dict[str, Union[TestDesciptor, BitstrmTestDescriptor]]]: + testDesciptor = dict[ + str, dict[str, Union[TestDesciptor, BitstrmTestDescriptor]] + ]() + for tag in IVAS_Bins.keys(): + testDesciptor[tag] = dict[ + str, Union[TestDesciptor, BitstrmTestDescriptor] + ]() + + IvasDecCmdMap = dict() + IsarDecCmdMap = dict() + for command in self.Commands["REND"]: + pyTestTag = self.getRendPyTestTag(command) + refCmdline = self.setCommandExec( + tag="REND", command=self.reformatCommand(command, ref=True), ref=True + ) + dutCmdline = self.setCommandExec( + tag="REND", command=self.reformatCommand(command) + ) + refOutput = self.getRendOutputFile(refCmdline) + dutOutput = self.getRendOutputFile(dutCmdline) + testDesciptor["REND"][pyTestTag] = TestDesciptor( + rawCmdline=command, + refOutput=refOutput, + dutOutput=dutOutput, + dutCmdline=dutCmdline, + refCmdline=refCmdline, + ) + for command in self.Commands["DEC"]: + pyTestTag = self.getPcmPytestTag(command) + refCmdline = self.setCommandExec( + tag="DEC", command=self.reformatCommand(command, ref=True), ref=True + ) + dutCmdline = self.setCommandExec( + tag="DEC", command=self.reformatCommand(command) + ) + refOutput = self.getOutputFile(refCmdline) + dutOutput = self.getOutputFile(dutCmdline) + testDesciptor["DEC"][pyTestTag] = TestDesciptor( + rawCmdline=command, + refOutput=refOutput, + dutOutput=dutOutput, + dutCmdline=dutCmdline, + refCmdline=refCmdline, + ) + IvasDecCmdMap[pyTestTag] = command + for command in self.Commands["ISAR"]: + pyTestTag = self.getIsarDecPytestTag(command) + refCmdline = self.setCommandExec( + tag="ISAR", command=self.reformatCommand(command, ref=True), ref=True + ) + dutCmdline = self.setCommandExec( + tag="ISAR", command=self.reformatCommand(command) + ) + refOutput = self.getRendOutputFile(refCmdline) + dutOutput = self.getRendOutputFile(dutCmdline) + testDesciptor["ISAR"][pyTestTag] = TestDesciptor( + rawCmdline=command, + refOutput=refOutput, + dutOutput=dutOutput, + dutCmdline=dutCmdline, + refCmdline=refCmdline, + ) + IsarDecCmdMap[pyTestTag] = command + for command in self.Commands["ENC"]: + pyTestTag = self.getEncPytestTag(command) + refCmdline = self.setCommandExec( + tag="ENC", command=self.reformatCommand(command, ref=True), ref=True + ) + dutCmdline = self.setCommandExec( + tag="ENC", command=self.reformatCommand(command) + ) + refOutput = self.getOutputFile(refCmdline) + dutOutput = self.getOutputFile(dutCmdline) + assert ".192" in dutOutput, "Output file not identified in dut" + assert ".192" in refOutput, "Output file not identified in ref" + if pyTestTag in IvasDecCmdMap.keys(): + rawDecCmdline = IvasDecCmdMap[pyTestTag] + refDecCmdline = self.setCommandExec( + tag="DEC", + command=self.reformatCommand(rawDecCmdline, ref=True), + ref=True, + ) + dutDecCmdline = self.setCommandExec( + tag="DEC", command=self.reformatCommand(rawDecCmdline) + ) + testDesciptor["ENC"][pyTestTag] = BitstrmTestDescriptor( + rawCmdline=command, + rawDecCmdline=rawDecCmdline, + refOutput=refOutput, + dutOutput=dutOutput, + dutCmdline=dutCmdline, + refCmdline=refCmdline, + dutDecCmdline=dutDecCmdline, + refDecCmdline=refDecCmdline, ) - if self.args.verbose: - print( - f"{encTag} {encoderPyTestTags[encTag]} -> {decoderPyTestTags[encTag]}" - ) - print(f"{self.Commands['ENC'][encoderPyTestTags[encTag]]}") - print(f"{self.Commands['DEC'][decoderPyTestTags[encTag]]}") else: - print(f"{encTag} not fount in decoder") - print( - f"Mapped decoder tests for {len(self.EncoderToDecoderCmdMap)} encoder tests out of {len(self.Commands['ENC'])} tests" - ) - assert len(self.EncoderToDecoderCmdMap) == len( - self.Commands["ENC"] - ), "Failed to Map Encoder Commands to Decoder Commands" - - def genEncoderReferences(self, command: str, encCommandIdx: int): - # RUN ENCODER COMMAND LINE WITH REFERENCE ENCODER - refCommand = self.reformatCommand(command=command, ref=True) - refEncOutput = self.getOutputFile(refCommand) - if not os.path.exists(refEncOutput): - self.process( - command=self.setCommandExec(tag="ENC", command=refCommand, ref=True) + print(f"{pyTestTag} not found in decoder") + for command in self.Commands["ISAR_ENC"]: + pyTestTag = self.getIsarEncPytestTag(command) + refCmdline = self.setCommandExec( + tag="ISAR_ENC", + command=self.reformatCommand(command, ref=True), + ref=True, + ) + dutCmdline = self.setCommandExec( + tag="ISAR_ENC", command=self.reformatCommand(command) ) + refOutput = self.getOutputFile(refCmdline) + dutOutput = self.getOutputFile(dutCmdline) + if pyTestTag in IsarDecCmdMap.keys(): + rawDecCmdline = IsarDecCmdMap[pyTestTag] + refDecCmdline = self.setCommandExec( + tag="ISAR", + command=self.reformatCommand(rawDecCmdline, ref=True), + ref=True, + ) + dutDecCmdline = self.setCommandExec( + tag="ISAR", command=self.reformatCommand(rawDecCmdline) + ) + testDesciptor["ISAR_ENC"][pyTestTag] = BitstrmTestDescriptor( + rawCmdline=command, + rawDecCmdline=rawDecCmdline, + refOutput=refOutput, + dutOutput=dutOutput, + dutCmdline=dutCmdline, + refCmdline=refCmdline, + dutDecCmdline=dutDecCmdline, + refDecCmdline=refDecCmdline, + ) + else: + print(f"{pyTestTag} not found in ISAR decoder") + print("No of tests :") + for tag in testDesciptor.keys(): + print(f" {tag} : {len(testDesciptor[tag])}") - # FIND CORRESPONDING DECODER COMMAND - decCommandIdx = self.EncoderToDecoderCmdMap[encCommandIdx] - refDecOutputFile = refEncOutput.replace(".192", "_REFDECODED.wav") + return testDesciptor - command = self.reformatCommand( - command=self.Commands["DEC"][decCommandIdx], ref=True - ) - command = command.replace("-VOIP", "") - refDecCmd = ( - [self.RefBins["DEC"]] - + command.split()[1:-2] - + [refEncOutput, refDecOutputFile] - ) - self.process(command=" ".join(refDecCmd)) - self.executedTests.value += 1 + def genEncoderReferences(self, tag: str, encPytestTag: str): + # RUN ENCODER'S OUTPUT DECODED WITH REF DECODER + testDesc = self.TestDesc[tag][encPytestTag] + assert isinstance( + testDesc, BitstrmTestDescriptor + ), f"Expected bitstream test descriptor for {tag}" + + # Decode the encoded output with Reference IVAS decoder + if tag == "ENC": + refDecOutputFile = testDesc.refOutput.replace(".192", "_REFDECODED.wav") + refDecCmd = testDesc.refDecCmdline.split()[:-2] + [ + testDesc.refOutput, + refDecOutputFile, + ] + else: + refDecOutputFile = testDesc.refOutput.replace(".splt.bit", ".wav") + refDecCmd = testDesc.refDecCmdline.split() + for idx, cmd in enumerate(refDecCmd): + if cmd == "-o" and (idx + 1) < len(refDecCmd): + refDecCmd[idx + 1] = refDecOutputFile + break + refDecCmd = ["" if x == "-VOIP" else x for x in refDecCmd] + refDecCmd = " ".join(refDecCmd) + self.process(command=refDecCmd) self.stats() - def runReferenceGeneration(self): - processes = list() # Multiprocess list - commands = conformance.Commands["ENC"] - self.totalTests = len(commands) + def runReferenceGeneration(self, encTag="ENC"): + selectedTests = list(self.TestDesc[encTag].keys()) + self.totalTests = len(selectedTests) if not self.args.no_multi_processing: - for commandIdx, command in enumerate(commands): - p = Process( - target=self.genEncoderReferences, args=(command, commandIdx) - ) - processes.append(p) - p.start() - for p in processes: - p.join() + with Pool() as pool: + args = [(encTag, pyTestsTag) for pyTestsTag in selectedTests] + pool.starmap(self.genEncoderReferences, args) else: - for commandIdx, command in enumerate(commands): - conformance.genEncoderReferences(command, commandIdx) - - def runOneEncoderTest(self, command: str): - encPytestTag = self.getEncPytestTag(command) - refEncOutput = self.getOutputFile(command) - refEncOutput = refEncOutput.replace( - "$CUT_PATH/ref/param_file/enc/", - f"{self.testvecDir}/testv/ref/param_file/enc/", - ) - refEncOutput = refEncOutput.replace( - "$CUT_PATH/ref/sba_bs/pkt/", - f"{self.testvecDir}/testv/ref/sba_bs/pkt/", - ) - refDecOutputFile = refEncOutput.replace(".192", "_REFDECODED.wav") - - # Run CUT Encoder - encCommandIdx = self.Commands["ENC"].index(command) - command = self.reformatCommand(command=command, ref=False) - command = self.setCommandExec(tag="ENC", command=command, ref=False) - dutEncOutput = self.getOutputFile(command) - self.process(command=command) - assert ".192" in dutEncOutput, "Output file not identified" - - # Decode the encoded output with Reference decoder - dutDecOutputFile = dutEncOutput.replace(".192", "_CUT_REFDECODED.wav") - decCommandIdx = self.EncoderToDecoderCmdMap[encCommandIdx] - command = self.reformatCommand( - command=self.Commands["DEC"][decCommandIdx], ref=False - ) - command = command.replace("-VOIP", "") - dutDecCmd = ( - [self.RefBins["DEC"]] - + command.split()[1:-2] - + [dutEncOutput, dutDecOutputFile] - ) - self.process(command=" ".join(dutDecCmd)) - self.mld( - "ENC", encPytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile - ) + for pyTestsTag in selectedTests: + self.genEncoderReferences(encTag, pyTestsTag) + + def analyseWavOutputTest(self, tag: str, dutPytestTag: str): + testDesc = self.TestDesc[tag][dutPytestTag] + assert isinstance( + testDesc, TestDesciptor + ), f"Expected pcm test descriptor for {tag}" + + if self.args.be_test: + DUTmdFiles = self.getMDfileList(outFile=testDesc.dutOutput) + REFmdFiles = self.getMDfileList(outFile=testDesc.refOutput) + self.beTest(tag, dutPytestTag, refFile=testDesc.refOutput, dutFile=testDesc.dutOutput, DUTmdFileList=DUTmdFiles, REFmdFileList=REFmdFiles ) + else: + ##### skip MLD verification for files with only 1 frame as MLD does not run with such files. Possible solution: append 0s and then compare ##### + if testDesc.rawCmdline.find("_cut.192.fer") == -1: + self.mld( + tag, + dutPytestTag, + refFile=testDesc.refOutput, + dutFile=testDesc.dutOutput, + ) - def runOneDecoderTest(self, tag: str, command: str): - dutPytestTag = self.getPcmPytestTag(command) - refInputFile = command.split()[-2].replace( - "$REF_PATH/ref", f"{self.testvDir}/ref" - ) - # refInputFile = refInputFile.replace("_cut.192.fer", ".192") - # refInputFile = refInputFile.replace(".fer.192", ".192").replace(".192.fer", ".192").replace("_cut.192.fer", ".192").replace("_cut.192", ".192") - refDecOutput = self.getOutputFile(command).replace( - "$CUT_PATH/ref", f"{self.testvDir}/ref" - ) - command = self.reformatCommand(command=command, ref=False) - # command = command.replace("-VOIP", "") - dutDecOutputFile = self.getOutputFile(command) - dutDecCmd = ( - [self.CutBins["DEC"]] - + command.split()[1:-2] - + [refInputFile, dutDecOutputFile] - ) - self.process(command=" ".join(dutDecCmd)) + def analyseOneEncoderTest(self, tag: str, encPytestTag: str): + testDesc = self.TestDesc[tag][encPytestTag] + assert isinstance( + testDesc, BitstrmTestDescriptor + ), f"Expected bitstream test descriptor for {tag}" - ##### skip MLD verification for files with only 1 frame as MLD does not run with such files. Possible solution: append 0s and then compare ##### - if refInputFile.find("_cut.192.fer") == -1: - self.mld( - "DEC", dutPytestTag, refFile=refDecOutput, dutFile=dutDecOutputFile - ) + if self.args.be_test: + self.beTest(tag, encPytestTag, refFile=testDesc.refOutput, dutFile=testDesc.dutOutput ) + else: + refDecOutputFile = testDesc.refOutput.replace(".192", "_REFDECODED.wav") + + dutDecOutputFile = testDesc.dutOutput.replace(".192", "_CUT_REFDECODED.wav") + # Decode the encoded output with Reference IVAS decoder + dutDecCmd = testDesc.refDecCmdline.split()[:-2] + [ + testDesc.dutOutput, + dutDecOutputFile, + ] + dutDecCmd = ["" if x == "-VOIP" else x for x in dutDecCmd] + dutDecCmd = " ".join(dutDecCmd) + self.process(command=dutDecCmd) + self.mld(tag, encPytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile) + + def analyseOneIsarEncoderTest(self, tag: str, pytestTag: str): + testDesc = self.TestDesc[tag][pytestTag] + assert isinstance( + testDesc, BitstrmTestDescriptor + ), f"Expected bitstream test descriptor for {tag}" + + if self.args.be_test: + self.beTest(tag, pytestTag, refFile=testDesc.refOutput, dutFile=testDesc.dutOutput ) + else: + refDecOutputFile = testDesc.refOutput.replace(".splt.bit", ".wav") + dutDecOutputFile = testDesc.dutOutput.replace(".splt.bit", ".wav") + # Decode the encoded output with Reference ISAR decoder + dutDecCmd = testDesc.refDecCmdline.split() + for idx, cmd in enumerate(dutDecCmd): + if cmd == "-o" and (idx + 1) < len(dutDecCmd): + dutDecCmd[idx + 1] = dutDecOutputFile + if cmd == "-i" and (idx + 1) < len(dutDecCmd): + dutDecCmd[idx + 1] = testDesc.dutOutput + self.process(command=" ".join(dutDecCmd)) + self.mld(tag, pytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile) def getRendOutputFile(self, command: str): cmds = command.split() @@ -296,45 +570,24 @@ class MLDConformance: return cmds[idx + 1] assert False, "Outputname not found" - def runOneRendererTest(self, tag: str, command: str): - refRendOutputFile = self.getRendOutputFile(command).replace( - "$CUT_PATH/renderer_short", f"{self.testvDir}/renderer_short" - ) - rendPytestTag = os.path.basename(refRendOutputFile).split(".")[-2] - command = self.reformatCommand(command=command, ref=False) - dutRendCmd = " ".join([self.CutBins["REND"]] + command.split()[1:]) - dutRendOutputFile = self.getRendOutputFile(dutRendCmd) - self.process(command=dutRendCmd) - self.mld( - "REND", rendPytestTag, refFile=refRendOutputFile, dutFile=dutRendOutputFile - ) - def getOutputFile(self, command: str): return command.split()[-1] - def setCommandExec(self, tag: str, command, ref: bool): + def getMDfileList(self, outFile: str): + MDfiles = [] + for i in range(0,3): + MDfiles.append(outFile + '.' + str(i) + '.csv') + MDfiles.append(outFile + '.met') + return MDfiles + + + def setCommandExec(self, tag: str, command, ref: bool = False): exec = self.RefBins[tag] if ref else self.CutBins[tag] commands = command.split() return " ".join([exec, *commands[1:]]) - def reformatCommand(self, command: str, ref: bool) -> str: - command = command.replace("$TESTV_PATH", self.scriptsDir) - command = command.replace( - "$REF_PATH/split_rendering", f"{self.testvecDir}/testv/split_rendering" - ) - - ################ HACKS ######################### - command = command.replace("_cut.192.fer", ".192") - command = command.replace("_cut.192", ".192") - command = command.replace(".fer.192", ".192") - command = command.replace(".192.fer", ".192") - ################################################## - command = command.replace( - "$REF_PATH/ref/param_file/", f"{self.testvDir}/ref/param_file/" - ) - command = command.replace( - "$REF_PATH/ref/sba_bs/pkt/", f"{self.testvDir}/ref/sba_bs/pkt/" - ) + def reformatCommand(self, command: str, ref: bool = False) -> str: + command = command.replace("$TESTV_PATH", self.testvecDir) if ref: command = command.replace( "$CUT_PATH/dut/sba_bs/pkt/", f"{self.testvDir}/ref/sba_bs/pkt/" @@ -346,87 +599,125 @@ class MLDConformance: "$CUT_PATH/ref/param_file/", f"{self.testvDir}/ref/param_file/" ) command = command.replace( - "$CUT_PATH/renderer_short/ref/", f"{self.testvDir}/ref/renderer_short/" + "$CUT_PATH/renderer_short/ref/", f"{self.testvDir}/renderer_short/ref/" ) command = command.replace( - "$CUT_PATH/split_rendering/cut/", - f"{self.testvDir}/ref/split_rendering/", + "$CUT_PATH/split_rendering/ref", + f"{self.testvDir}/split_rendering/ref", ) command = command.replace( "$CUT_PATH/ref/sba_bs/", f"{self.testvDir}/ref/sba_bs/" ) else: - #command = command.replace( - # "$CUT_PATH/dut/sba_bs/pkt/", f"{self.outputDir}/dut/enc/" - #) command = command.replace( - "$CUT_PATH/ref/param_file/enc/", f"{self.outputDir}/dut/enc/" + "$CUT_PATH/ref/param_file/enc/", f"{self.outputDir}/enc/" ) command = command.replace( - "$CUT_PATH/ref/param_file/dec/", f"{self.outputDir}/dut/dec/" + "$CUT_PATH/ref/param_file/dec/", f"{self.outputDir}/dec/" ) command = command.replace( - "$CUT_PATH/renderer_short/ref/", f"{self.outputDir}/dut/renderer_short/" + "$CUT_PATH/renderer_short/ref/", f"{self.outputDir}/renderer_short/ref" ) command = command.replace( "$CUT_PATH/split_rendering/cut/", - f"{self.outputDir}/dut/split_rendering/", + f"{self.outputDir}/split_rendering/cut/", + ) + command = command.replace( + "$CUT_PATH/split_rendering/ref", + f"{self.outputDir}/split_rendering/", ) command = command.replace( - "$CUT_PATH/ref/sba_bs/pkt/", f"{self.outputDir}/dut/enc/" + "$CUT_PATH/ref/sba_bs/pkt/", f"{self.outputDir}/enc/" ) command = command.replace( - "$CUT_PATH/ref/sba_bs/raw/", f"{self.outputDir}/dut/dec/" + "$CUT_PATH/ref/sba_bs/raw/", f"{self.outputDir}/dec/" ) + command = command.replace( + "$REF_PATH/split_rendering", f"{self.testvecDir}/testv/split_rendering" + ) + command = command.replace( + "$REF_PATH/ref/param_file/", f"{self.testvDir}/ref/param_file/" + ) + command = command.replace( + "$REF_PATH/ref/sba_bs/pkt/", f"{self.testvDir}/ref/sba_bs/pkt/" + ) return command - def runOneCommand(self, tag: str, command: str): + def runOneCommand(self, tag: str, pyTestsTag: str): + # Run CUT Cmdline + testDesc = self.TestDesc[tag][pyTestsTag] + self.process(command=testDesc.dutCmdline) + self.stats() + + def analyseOneCommand(self, tag: str, pyTestsTag: str): if tag == "ENC": - self.runOneEncoderTest(command) + self.analyseOneEncoderTest(tag, pyTestsTag) elif tag == "DEC": - self.runOneDecoderTest(tag, command) + self.analyseWavOutputTest(tag, pyTestsTag) elif tag == "REND": - self.runOneRendererTest(tag, command) + self.analyseWavOutputTest(tag, pyTestsTag) + elif tag == "ISAR_ENC": + self.analyseOneIsarEncoderTest(tag, pyTestsTag) + elif tag == "ISAR": + self.analyseWavOutputTest(tag, pyTestsTag) else: assert False, f"Un-implemented Tag {tag}" - self.executedTests.value += 1 self.stats() def runTag(self, tag: str): - self.executedTests.value = 0 - self.failedTests.value = 0 - # reset MLD, Sample Stats - open(self.mldcsv[tag], "w").close() - with open(self.sampleStats[tag], "w") as f: - f.write(f"PYTESTTAG, MAXDIFF, RMSdB, BEFRAMES_PERCENT, MAX_MLD\n") - - processes = list() # Multiprocess list - commands = list() + selectedTests = list() if self.filter: - for command in self.Commands[tag]: - if self.filter in command: - commands.append(command) + for pyTestsTag in self.TestDesc[tag].keys(): + if self.filter in self.TestDesc[tag][pyTestsTag].rawCmdline: + selectedTests.append(pyTestsTag) else: - commands = self.Commands[tag] + selectedTests = list(self.TestDesc[tag].keys()) - self.totalTests = len(commands) + self.totalTests = len(selectedTests) print( f"Executing tests for {tag} {'Filter='+self.filter if self.filter else ''} ({self.totalTests} tests)" ) if not self.args.no_multi_processing: - for command in commands: - p = Process( - target=self.runOneCommand, - args=(tag, command), - ) - processes.append(p) - p.start() - for p in processes: - p.join() + with Pool() as pool: + args = [(tag, pyTestsTag) for pyTestsTag in selectedTests] + pool.starmap(self.runOneCommand, args) + else: + for pyTestsTag in selectedTests: + self.runOneCommand(tag, pyTestsTag) + + def analyseTag(self, tag: str): + # reset MLD, Sample Stats + if self.args.be_test: + with open(self.BEcsv[tag], "w") as f: + f.write(f"PYTESTTAG, BE=0 NON-BE=1\n") + else: + open(self.mldcsv[tag], "w").close() + with open(self.sampleStats[tag], "w") as f: + f.write(f"PYTESTTAG, MAXDIFF, RMSdB, BEFRAMES_PERCENT, MAX_MLD\n") + selectedTests = list() + if self.filter: + for pyTestsTag in self.TestDesc[tag].keys(): + if self.filter in self.TestDesc[tag][pyTestsTag].rawCmdline: + selectedTests.append(pyTestsTag) else: - for command in commands: - self.runOneCommand(tag, command) + selectedTests = list(self.TestDesc[tag].keys()) + + self.totalTests = len(selectedTests) + print( + f"Analysing tests for {tag} {'Filter='+self.filter if self.filter else ''} ({self.totalTests} tests)" + ) + if not self.args.no_multi_processing: + with Pool() as pool: + args = [(tag, pyTestsTag) for pyTestsTag in selectedTests] + pool.starmap(self.analyseOneCommand, args) + else: + for pyTestsTag in selectedTests: + self.analyseOneCommand(tag, pyTestsTag) + if self.args.be_test: + self.doBEanalysis(selectTag=tag) + else: + self.doAnalysis(selectTag=tag) def process(self, command) -> int: if self.args.verbose: @@ -442,15 +733,11 @@ class MLDConformance: if c.returncode: with open(self.failedCmdsFile, "a") as f: f.write(command + "\n") - self.failedTests.value += 1 # c.check_returncode() return 0 def stats(self): - print( - f"Executed: {self.executedTests.value} / {self.totalTests} Failed: {self.failedTests.value}", - end="\r", - ) + pass def getSampleStats(self, refSamples: np.ndarray, dutSamples: np.ndarray): nSamples = min(refSamples.shape[0], dutSamples.shape[0]) @@ -467,6 +754,13 @@ class MLDConformance: def mld(self, tag, pytestTag, refFile, dutFile): mldThisFile = np.zeros(0) + with open(self.failedCmdsFile, "a") as f: + if not os.path.exists(refFile): + f.write(f"File does not exists: {refFile}\n") + return + if not os.path.exists(dutFile): + f.write(f"File does not exists : {dutFile}\n") + return with tempfile.TemporaryDirectory() as tmpdir: refSamples, fsR = readfile(refFile, outdtype="float") dutSamples, fsD = readfile(dutFile, outdtype="float") @@ -486,26 +780,40 @@ class MLDConformance: for ch in range(nChans): mldFile = os.path.join( - tmpdir, f"{tempfile.gettempprefix()}_ch{ch}_MLD.csv" + tmpdir, f"{tempfile.gettempprefix()}_ch{ch}_MLD2.txt" ) refFileMono = os.path.join( - tmpdir, os.path.basename(refFile).replace(".wav", f"_ch{ch}.wav") + tmpdir, + os.path.basename(refFile).replace(".wav", f"_REF_ch{ch}.wav"), ) dutFileMono = os.path.join( - tmpdir, os.path.basename(dutFile).replace(".wav", f"_ch{ch}.wav") + tmpdir, + os.path.basename(dutFile).replace(".wav", f"_DUT_ch{ch}.wav"), ) writefile(refFileMono, refSamples[:, ch], 48000) writefile(dutFileMono, dutSamples[:, ch], 48000) + command = [ - self.mldbin, - "-o", - mldFile, + self.wavdiffbin, "-s", refFileMono, dutFileMono, ] - self.process(" ".join(command)) - mldThisChan = np.loadtxt(mldFile, delimiter=" ", dtype=float) + + # Output on stdout with wavdiffbin + with open(mldFile, "w") as fd: + c = subprocess.run( + " ".join(command), + stdout=fd, + stderr=subprocess.STDOUT, + text=True, + shell=True, + ) + + mldThisChan = np.loadtxt( + mldFile, delimiter=";", dtype=float, skiprows=1 + ) + mldThisChan = mldThisChan[:, 2] if ch == 0: mldThisFile = mldThisChan else: @@ -519,15 +827,68 @@ class MLDConformance: f"{pytestTag}, {maxDiff}, {rmsdB}, {beSamplesPercent}, {mldThisFile.max()}\n" ) + def beTest(self, tag, pytestTag, refFile, dutFile, DUTmdFileList=[], REFmdFileList=[]): + BE_flag = 0 + if not filecmp.cmp(refFile, dutFile): + BE_flag = 1 + with open(self.BEcsv[tag], "a") as f: + f.write( + f"{pytestTag}, {BE_flag}\n" + ) + + for i in range(0,len(DUTmdFileList)): + if os.path.exists(DUTmdFileList[i]): + BE_flag = 0 + if not filecmp.cmp(REFmdFileList[i], DUTmdFileList[i]): + BE_flag = 1 + with open(self.BEcsv[tag], "a") as f: + f.write( + f"{DUTmdFileList[i]}, {BE_flag}\n" + ) + + + def doBEanalysis(self, selectTag="all"): + keys = IVAS_Bins.keys() if selectTag == "all" else [selectTag] + for tag in keys: + if os.path.exists(self.BEcsv[tag]): + BEresult = np.loadtxt( + self.BEcsv[tag], + delimiter=",", + dtype=int, + skiprows=1, + usecols=1, + ) + if np.sum(BEresult) > 0: + print(f"<{tag}> FAILED BE TEST, check {self.BEcsv[tag]}") + else: + print(f"<{tag}> PASSED BE TEST") + def doAnalysis(self, selectTag="all"): - keys = MLDConformance.IVAS_Bins.keys() if selectTag == "all" else [selectTag] + keys = IVAS_Bins.keys() if selectTag == "all" else [selectTag] for tag in keys: if os.path.exists(self.mldcsv[tag]): mdlValues = np.loadtxt(self.mldcsv[tag], delimiter=" ", dtype=float) + bePercent = np.loadtxt( + self.sampleStats[tag], + delimiter=",", + dtype=float, + skiprows=1, + usecols=3, + ) + maxDiff = np.loadtxt( + self.sampleStats[tag], + delimiter=",", + dtype=float, + skiprows=1, + usecols=1, + ) + bePercentAvg = np.average(bePercent) + maxDiffmax = np.max(maxDiff) * 32768.0 N = mdlValues.shape[0] if N == 0: continue m0 = np.sum(mdlValues == 0) + m05 = np.sum(mdlValues <= 0.5) m1 = np.sum(mdlValues <= 1.0) m2 = np.sum(mdlValues <= 2.0) m5 = np.sum(mdlValues <= 5.0) @@ -537,9 +898,14 @@ class MLDConformance: print(f"<{tag}> Total Frames: {N}") print(f"<{tag}> MAX MLD across all frames : {mdlValues.max()}") print(f"<{tag}> Frames with MLD == 0 : {m0} frames ({PCNT(m0)}%)") + print(f"<{tag}> Frames with MLD <= 0.5 : {m05} frames ({PCNT(m05)}%)") print(f"<{tag}> Frames with MLD <= 1 : {m1} frames ({PCNT(m1)}%)") print(f"<{tag}> Frames with MLD <= 2 : {m2} frames ({PCNT(m2)}%)") print(f"<{tag}> Frames with MLD <= 5 : {m5} frames ({PCNT(m5)}%)") + print(f"<{tag}> BE samples percentage = {bePercentAvg}") + print( + f"<{tag}> max absolute diff = {maxDiffmax}, sample range (-32768, 32767)" + ) print("##########################################################\n") @@ -557,11 +923,13 @@ if __name__ == "__main__": parser.add_argument( "--ref_build_path", type=str, + default="", help="Path to the reference build folder containing IVAS Encoder, Decoder, Renderer and Post Render binaries", ) parser.add_argument( "--cut_build_path", type=str, + default="", help="Path to the CUT build folder containing IVAS Encoder, Decoder, Renderer and Post Render binaries", ) parser.add_argument( @@ -570,6 +938,12 @@ if __name__ == "__main__": action="store_true", help="Regenerate the encoder reference bitstreams and decoded outputs", ) + parser.add_argument( + "--dec-for-dut-enc", + default=False, + action="store_true", + help="process dut ENC files with REF decoder", + ) parser.add_argument( "--verbose", default=False, @@ -593,8 +967,14 @@ if __name__ == "__main__": "--test-mode", type=str, default="ALL", - help='Choose tests to run ["ENC", "DEC", "REND", "ISAR", "ALL"]', + help='Choose tests to run ["ENC", "DEC", "REND", "ISAR", "ISAR_ENC", "ALL"]', ) + parser.add_argument( + "--be-test", + default=False, + action="store_true", + help='runs only BE tests', + ) parser.add_argument( "--no-multi-processing", default=False, @@ -602,29 +982,39 @@ if __name__ == "__main__": help="Disable multi-processing for sequential test run (debugging)", ) parser.add_argument( - "--analyse-only", + "--analyse", + default=False, + action="store_true", + help="Perform MLD analysis on CUT outputs generated", + ) + parser.add_argument( + "--report-only", + default=False, + action="store_true", + help="Do not run DUT, use existing mld and bitdiff stats files to generate analysis only", + ) + parser.add_argument( + "-c", + "--clean-output-dir", default=False, action="store_true", help="Do not run DUT, use existing mld and bitdiff stats files to generate analysis only", ) - args = parser.parse_args() conformance = MLDConformance(args) - conformance.accumulateCommands() if args.regenerate_enc_refs: - conformance.runReferenceGeneration() + conformance.runReferenceGeneration(encTag="ISAR_ENC") + conformance.runReferenceGeneration(encTag="ENC") sys.exit(0) - testTags = ( - MLDConformance.IVAS_Bins.keys() if args.test_mode == "ALL" else [args.test_mode] - ) + testTags = IVAS_Bins.keys() if args.test_mode == "ALL" else [args.test_mode] for tag in testTags: - if tag == "ISAR": - # Not implemented yet - continue - if not args.analyse_only: + if args.report_only: + conformance.doAnalysis(selectTag=tag) + elif not args.analyse: conformance.runTag(tag) - conformance.doAnalysis(selectTag=tag) + else: + conformance.analyseTag(tag)