From 2b0ee9936a35ccc4429fb1b75dd1931c1c215a99 Mon Sep 17 00:00:00 2001 From: Ripinder Singh Date: Tue, 4 Nov 2025 14:21:25 +1100 Subject: [PATCH 01/16] ISAR Conformance tests Signed-off-by: Ripinder Singh --- scripts/ivas_conformance/runConformance.py | 204 +++++++++++++++------ 1 file changed, 150 insertions(+), 54 deletions(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 15f9fbcd53..760be1674a 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -52,12 +52,14 @@ class MLDConformance: "ENC": "IVAS_cod", "DEC": "IVAS_dec", "REND": "IVAS_rend", + "ISAR_ENC": "IVAS_dec", "ISAR": "ISAR_post_rend", } def setupCommon(self): self.Commands = dict() self.EncoderToDecoderCmdMap = dict() + self.IsarEncoderToDecoderCmdMap = dict() for tag in MLDConformance.IVAS_Bins.keys(): self.Commands[tag] = list() @@ -68,7 +70,7 @@ class MLDConformance: os.makedirs(self.outputDir, exist_ok=True) subdirs = ["enc", "dec", "renderer_short", "split_rendering"] for odir in subdirs: - os.makedirs(os.path.join(self.outputDir, "ref", odir), exist_ok=True) + os.makedirs(os.path.join(self.testvDir, "ref", odir), exist_ok=True) os.makedirs(os.path.join(self.outputDir, "dut", odir), exist_ok=True) self.logFile = os.path.join(self.outputDir, "runlog.txt") @@ -121,25 +123,23 @@ class MLDConformance: for root, _, files in os.walk(self.testvecDir): for file_name in files: basename, ext = os.path.splitext(file_name) - if ( - ("Readme_IVAS_" in basename) - and ext == ".txt" - and not ("ISAR" in basename) - ): + if ("Readme_IVAS_" in basename) and ext == ".txt": print(f"Accumulating commands from {file_name}") file = os.path.join(root, file_name) self.parseCommandsFile(file) self.mapEncoderToDecoderCommands() + self.mapIsarEncToDecCommands() print("No of tests :") for key in self.Commands.keys(): print(f" {key} : {len(self.Commands[key])}") def parseCommandsFile(self, filePath): + isISAREnc = "IVAS_ISAR_dec" in filePath with open(filePath) as fp: for line in fp.readlines(): m = re.search(r"^\$(CUT_.+_BIN) ", line) if m: - tag = m.group(1).split("_")[1] + tag = m.group(1).split("_")[1] if not isISAREnc else "ISAR_ENC" if tag in self.Commands.keys(): self.Commands[tag].append(line) @@ -151,9 +151,48 @@ class MLDConformance: ) return decInput.split(".")[-2] + def getIsarDecPytestTag(self, command: str) -> str: + getName = False + for command in command.split(): + if getName: + return os.path.basename(command).split(".")[-3] + getName = True if command == "-i" else getName + assert False, f"No match found for {command}" + def getEncPytestTag(self, command: str) -> str: return os.path.basename(command.split()[-1]).split(".")[-2] + def getIsarEncPytestTag(self, command: str) -> str: + return os.path.basename(command.split()[-1]).split(".")[-3] + + def mapIsarEncToDecCommands(self): + decoderPyTestTags = dict() + encoderPyTestTags = dict() + for idx, command in enumerate(self.Commands["ISAR"]): + decoderPyTestTags[self.getIsarDecPytestTag(command)] = idx + for idx, command in enumerate(self.Commands["ISAR_ENC"]): + encoderPyTestTags[self.getIsarEncPytestTag(command)] = idx + + for encTag in encoderPyTestTags.keys(): + if encTag in decoderPyTestTags.keys(): + self.IsarEncoderToDecoderCmdMap[encoderPyTestTags[encTag]] = ( + decoderPyTestTags[encTag] + ) + if self.args.verbose: + print( + f"{encTag} {encoderPyTestTags[encTag]} -> {decoderPyTestTags[encTag]}" + ) + print(f"{self.Commands['ISAR_ENC'][encoderPyTestTags[encTag]]}") + print(f"{self.Commands['ISAR'][decoderPyTestTags[encTag]]}") + else: + print(f"{encTag} not fount in decoder") + print( + f"Mapped decoder tests for {len(self.IsarEncoderToDecoderCmdMap)} encoder tests out of {len(self.Commands['ISAR_ENC'])} tests" + ) + assert len(self.IsarEncoderToDecoderCmdMap) == len( + self.Commands["ISAR_ENC"] + ), "Failed to Map Encoder Commands to Decoder Commands" + def mapEncoderToDecoderCommands(self): decoderPyTestTags = dict() encoderPyTestTags = dict() @@ -182,40 +221,52 @@ class MLDConformance: self.Commands["ENC"] ), "Failed to Map Encoder Commands to Decoder Commands" - def genEncoderReferences(self, command: str, encCommandIdx: int): + def genEncoderReferences( + self, command: str, encCommandIdx: int, encTag: str = "ENC" + ): # RUN ENCODER COMMAND LINE WITH REFERENCE ENCODER - refCommand = self.reformatCommand(command=command, ref=True) + refCommand = self.reformatCommand(command, ref=True) refEncOutput = self.getOutputFile(refCommand) - if not os.path.exists(refEncOutput): - self.process( - command=self.setCommandExec(tag="ENC", command=refCommand, ref=True) - ) + # if not os.path.exists(refEncOutput): + self.process( + command=self.setCommandExec(tag=encTag, command=refCommand, ref=True) + ) # FIND CORRESPONDING DECODER COMMAND - decCommandIdx = self.EncoderToDecoderCmdMap[encCommandIdx] - refDecOutputFile = refEncOutput.replace(".192", "_REFDECODED.wav") + if encTag == "ISAR_ENC": + decTag = "ISAR" + decCommandIdx = self.IsarEncoderToDecoderCmdMap[encCommandIdx] + refDecOutputFile = refEncOutput.replace(".splt.bit", ".splt.REFDECODED.wav") + else: + decTag = "DEC" + decCommandIdx = self.EncoderToDecoderCmdMap[encCommandIdx] + refDecOutputFile = refEncOutput.replace(".192", "_REFDECODED.wav") command = self.reformatCommand( - command=self.Commands["DEC"][decCommandIdx], ref=True + command=self.Commands[decTag][decCommandIdx], ref=True ) command = command.replace("-VOIP", "") - refDecCmd = ( - [self.RefBins["DEC"]] - + command.split()[1:-2] - + [refEncOutput, refDecOutputFile] - ) + if encTag == "ISAR_ENC": + refDecCmd = [self.RefBins[decTag]] + command.split()[1:] + else: + refDecCmd = ( + [self.RefBins[decTag]] + + command.split()[1:-2] + + [refEncOutput, refDecOutputFile] + ) + self.process(command=" ".join(refDecCmd)) self.executedTests.value += 1 self.stats() - def runReferenceGeneration(self): + def runReferenceGeneration(self, encTag="ENC"): processes = list() # Multiprocess list - commands = conformance.Commands["ENC"] + commands = conformance.Commands[encTag] self.totalTests = len(commands) if not self.args.no_multi_processing: for commandIdx, command in enumerate(commands): p = Process( - target=self.genEncoderReferences, args=(command, commandIdx) + target=self.genEncoderReferences, args=(command, commandIdx, encTag) ) processes.append(p) p.start() @@ -223,7 +274,7 @@ class MLDConformance: p.join() else: for commandIdx, command in enumerate(commands): - conformance.genEncoderReferences(command, commandIdx) + conformance.genEncoderReferences(command, commandIdx, encTag) def runOneEncoderTest(self, command: str): encPytestTag = self.getEncPytestTag(command) @@ -240,18 +291,16 @@ class MLDConformance: # Run CUT Encoder encCommandIdx = self.Commands["ENC"].index(command) - command = self.reformatCommand(command=command, ref=False) - command = self.setCommandExec(tag="ENC", command=command, ref=False) + command = self.reformatCommand(command) + command = self.setCommandExec(tag="ENC", command=command) dutEncOutput = self.getOutputFile(command) - self.process(command=command) + self.process(command) assert ".192" in dutEncOutput, "Output file not identified" # Decode the encoded output with Reference decoder dutDecOutputFile = dutEncOutput.replace(".192", "_CUT_REFDECODED.wav") decCommandIdx = self.EncoderToDecoderCmdMap[encCommandIdx] - command = self.reformatCommand( - command=self.Commands["DEC"][decCommandIdx], ref=False - ) + command = self.reformatCommand(command=self.Commands["DEC"][decCommandIdx]) command = command.replace("-VOIP", "") dutDecCmd = ( [self.RefBins["DEC"]] @@ -273,7 +322,7 @@ class MLDConformance: refDecOutput = self.getOutputFile(command).replace( "$CUT_PATH/ref", f"{self.testvDir}/ref" ) - command = self.reformatCommand(command=command, ref=False) + command = self.reformatCommand(command) # command = command.replace("-VOIP", "") dutDecOutputFile = self.getOutputFile(command) dutDecCmd = ( @@ -301,7 +350,7 @@ class MLDConformance: "$CUT_PATH/renderer_short", f"{self.testvDir}/renderer_short" ) rendPytestTag = os.path.basename(refRendOutputFile).split(".")[-2] - command = self.reformatCommand(command=command, ref=False) + command = self.reformatCommand(command) dutRendCmd = " ".join([self.CutBins["REND"]] + command.split()[1:]) dutRendOutputFile = self.getRendOutputFile(dutRendCmd) self.process(command=dutRendCmd) @@ -309,32 +358,67 @@ class MLDConformance: "REND", rendPytestTag, refFile=refRendOutputFile, dutFile=dutRendOutputFile ) + def runOneIsarEncoderTest(self, command: str): + encCommandIdx = self.Commands["ISAR_ENC"].index(command) + decCommandIdx = self.IsarEncoderToDecoderCmdMap[encCommandIdx] + + isarEncPytestTag = self.getIsarEncPytestTag(command) + refEncCommand = self.reformatCommand(command, ref=True) + refEncOutput = self.getOutputFile(refEncCommand) + refDecOutputFile = refEncOutput.replace(".splt.bit", ".wav") + + # Run CUT Encoder + dutEncCommand = self.reformatCommand(command) + dutEncOutput = self.getOutputFile(dutEncCommand) + self.process(command=self.setCommandExec(tag="ISAR_ENC", command=dutEncCommand)) + + # Decode the encoded output with Reference decoder + dutDecCommand = self.reformatCommand( + command=self.Commands["ISAR"][decCommandIdx] + ) + dutDecOutputFile = self.getRendOutputFile(dutDecCommand) + self.process( + command=self.setCommandExec(tag="ISAR", command=dutDecCommand, ref=True) + ) + self.mld( + "ISAR_ENC", + isarEncPytestTag, + refFile=refDecOutputFile, + dutFile=dutDecOutputFile, + ) + + def runOneIsarDecoderTest(self, command: str): + isarEncPytestTag = self.getIsarDecPytestTag(command) + refDecCommand = self.reformatCommand(command, ref=True) + refDecOutputFile = self.getRendOutputFile(refDecCommand) + + # Decode the encoded output with Reference decoder + dutDecCommand = self.reformatCommand(command) + dutDecOutputFile = self.getRendOutputFile(dutDecCommand) + self.process( + command=self.setCommandExec(tag="ISAR", command=dutDecCommand) + ) + self.mld( + "ISAR", isarEncPytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile + ) + def getOutputFile(self, command: str): return command.split()[-1] - def setCommandExec(self, tag: str, command, ref: bool): + def setCommandExec(self, tag: str, command, ref: bool = False): exec = self.RefBins[tag] if ref else self.CutBins[tag] commands = command.split() return " ".join([exec, *commands[1:]]) - def reformatCommand(self, command: str, ref: bool) -> str: + def reformatCommand(self, command: str, ref: bool = False) -> str: command = command.replace("$TESTV_PATH", self.scriptsDir) - command = command.replace( - "$REF_PATH/split_rendering", f"{self.testvecDir}/testv/split_rendering" - ) - ################ HACKS ######################### command = command.replace("_cut.192.fer", ".192") command = command.replace("_cut.192", ".192") command = command.replace(".fer.192", ".192") command = command.replace(".192.fer", ".192") ################################################## - command = command.replace( - "$REF_PATH/ref/param_file/", f"{self.testvDir}/ref/param_file/" - ) - command = command.replace( - "$REF_PATH/ref/sba_bs/pkt/", f"{self.testvDir}/ref/sba_bs/pkt/" - ) + if ref: command = command.replace( "$CUT_PATH/dut/sba_bs/pkt/", f"{self.testvDir}/ref/sba_bs/pkt/" @@ -349,16 +433,13 @@ class MLDConformance: "$CUT_PATH/renderer_short/ref/", f"{self.testvDir}/ref/renderer_short/" ) command = command.replace( - "$CUT_PATH/split_rendering/cut/", - f"{self.testvDir}/ref/split_rendering/", + "$CUT_PATH/split_rendering/ref", + f"{self.testvDir}/ref/split_rendering", ) command = command.replace( "$CUT_PATH/ref/sba_bs/", f"{self.testvDir}/ref/sba_bs/" ) else: - #command = command.replace( - # "$CUT_PATH/dut/sba_bs/pkt/", f"{self.outputDir}/dut/enc/" - #) command = command.replace( "$CUT_PATH/ref/param_file/enc/", f"{self.outputDir}/dut/enc/" ) @@ -372,6 +453,10 @@ class MLDConformance: "$CUT_PATH/split_rendering/cut/", f"{self.outputDir}/dut/split_rendering/", ) + command = command.replace( + "$CUT_PATH/split_rendering/ref", + f"{self.outputDir}/dut/split_rendering", + ) command = command.replace( "$CUT_PATH/ref/sba_bs/pkt/", f"{self.outputDir}/dut/enc/" ) @@ -379,6 +464,15 @@ class MLDConformance: "$CUT_PATH/ref/sba_bs/raw/", f"{self.outputDir}/dut/dec/" ) + command = command.replace( + "$REF_PATH/split_rendering", f"{self.testvecDir}/testv/split_rendering" + ) + command = command.replace( + "$REF_PATH/ref/param_file/", f"{self.testvDir}/ref/param_file/" + ) + command = command.replace( + "$REF_PATH/ref/sba_bs/pkt/", f"{self.testvDir}/ref/sba_bs/pkt/" + ) return command def runOneCommand(self, tag: str, command: str): @@ -388,6 +482,10 @@ class MLDConformance: self.runOneDecoderTest(tag, command) elif tag == "REND": self.runOneRendererTest(tag, command) + elif tag == "ISAR_ENC": + self.runOneIsarEncoderTest(command) + elif tag == "ISAR": + self.runOneIsarDecoderTest(command) else: assert False, f"Un-implemented Tag {tag}" self.executedTests.value += 1 @@ -615,16 +713,14 @@ if __name__ == "__main__": conformance.accumulateCommands() if args.regenerate_enc_refs: - conformance.runReferenceGeneration() + conformance.runReferenceGeneration(encTag="ISAR_ENC") + conformance.runReferenceGeneration(encTag="ENC") sys.exit(0) testTags = ( MLDConformance.IVAS_Bins.keys() if args.test_mode == "ALL" else [args.test_mode] ) for tag in testTags: - if tag == "ISAR": - # Not implemented yet - continue if not args.analyse_only: conformance.runTag(tag) conformance.doAnalysis(selectTag=tag) -- GitLab From 733a2882b09c8f90d8de992f975b81c6210f7213 Mon Sep 17 00:00:00 2001 From: rtyag Date: Wed, 5 Nov 2025 16:59:15 +1100 Subject: [PATCH 02/16] add BE, max diff stat to the print --- scripts/ivas_conformance/runConformance.py | 24 +++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 15f9fbcd53..e23b5218c2 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -64,7 +64,8 @@ class MLDConformance: # CREATE OUTPUT DIRECTORY STRUCTURE : CLEAN PREV OUTPUT self.outputDir = os.path.join(self.scriptsDir, "CUT_OUTPUTS") if os.path.exists(self.outputDir): - shutil.rmtree(self.outputDir, ignore_errors=False) + if not self.args.dec_for_dut_enc: + shutil.rmtree(self.outputDir, ignore_errors=False) os.makedirs(self.outputDir, exist_ok=True) subdirs = ["enc", "dec", "renderer_short", "split_rendering"] for odir in subdirs: @@ -79,7 +80,11 @@ class MLDConformance: def setupDUT(self): self.cut_build_path = args.cut_build_path self.filter = args.filter - self.mldbin = os.path.join(self.toolsdir, platform.system(), "mld") + exe_platform = platform.system() + if platform.system() == 'Windows': + exe_platform = 'Win32' + self.mldbin = os.path.join(self.toolsdir, exe_platform, "mld") + print(self.mldbin) self.CutBins = dict() self.mldcsv = dict() self.sampleStats = dict() @@ -243,7 +248,8 @@ class MLDConformance: command = self.reformatCommand(command=command, ref=False) command = self.setCommandExec(tag="ENC", command=command, ref=False) dutEncOutput = self.getOutputFile(command) - self.process(command=command) + if not self.args.dec_for_dut_enc: + self.process(command=command) assert ".192" in dutEncOutput, "Output file not identified" # Decode the encoded output with Reference decoder @@ -524,6 +530,10 @@ class MLDConformance: for tag in keys: if os.path.exists(self.mldcsv[tag]): mdlValues = np.loadtxt(self.mldcsv[tag], delimiter=" ", dtype=float) + bePercent = np.loadtxt(self.sampleStats[tag], delimiter=",", dtype=float, skiprows=1, usecols=3) + maxDiff = np.loadtxt(self.sampleStats[tag], delimiter=",", dtype=float, skiprows=1, usecols=1) + bePercentAvg = np.average(bePercent) + maxDiffmax = np.max(maxDiff)*32768.0 N = mdlValues.shape[0] if N == 0: continue @@ -540,6 +550,8 @@ class MLDConformance: print(f"<{tag}> Frames with MLD <= 1 : {m1} frames ({PCNT(m1)}%)") print(f"<{tag}> Frames with MLD <= 2 : {m2} frames ({PCNT(m2)}%)") print(f"<{tag}> Frames with MLD <= 5 : {m5} frames ({PCNT(m5)}%)") + print(f"<{tag}> BE frames percentage = {bePercentAvg}") + print(f"<{tag}> max absolute diff = {maxDiffmax}, sample range (-32768, 32767)") print("##########################################################\n") @@ -570,6 +582,12 @@ if __name__ == "__main__": action="store_true", help="Regenerate the encoder reference bitstreams and decoded outputs", ) + parser.add_argument( + "--dec-for-dut-enc", + default=False, + action="store_true", + help="process dut ENC files with REF decoder", + ) parser.add_argument( "--verbose", default=False, -- GitLab From 91c7e77f7c57a2836e41541da7fa659b0d388d19 Mon Sep 17 00:00:00 2001 From: Ripinder Singh Date: Wed, 5 Nov 2025 17:19:49 +1100 Subject: [PATCH 03/16] Fix for MLD overwritting files for REF and DUT to same temporary file Signed-off-by: Ripinder Singh --- scripts/ivas_conformance/runConformance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 760be1674a..67a07691a2 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -587,10 +587,10 @@ class MLDConformance: tmpdir, f"{tempfile.gettempprefix()}_ch{ch}_MLD.csv" ) refFileMono = os.path.join( - tmpdir, os.path.basename(refFile).replace(".wav", f"_ch{ch}.wav") + tmpdir, os.path.basename(refFile).replace(".wav", f"_REF_ch{ch}.wav") ) dutFileMono = os.path.join( - tmpdir, os.path.basename(dutFile).replace(".wav", f"_ch{ch}.wav") + tmpdir, os.path.basename(dutFile).replace(".wav", f"_DUT_ch{ch}.wav") ) writefile(refFileMono, refSamples[:, ch], 48000) writefile(dutFileMono, dutSamples[:, ch], 48000) -- GitLab From b5716a0cc3cafc2d1557b4c03fd77a7917454c2c Mon Sep 17 00:00:00 2001 From: Rishabh Tyagi Date: Wed, 5 Nov 2025 22:25:20 +1100 Subject: [PATCH 04/16] use wavdiff and remove dependency on 3d tools --- scripts/ivas_conformance/runConformance.py | 170 ++++++++++++++++++++- 1 file changed, 166 insertions(+), 4 deletions(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 15f9fbcd53..31df313e39 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -37,14 +37,153 @@ import numpy as np import subprocess import tempfile import sys -from typing import Optional +from typing import Tuple from multiprocessing import Process, Value import shutil +import scipy.io.wavfile as wav +import warnings +import math +import scipy.signal as sig sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) -from pyaudio3dtools.audiofile import readfile, writefile -from pyaudio3dtools.audioarray import resample +#from pyaudio3dtools.audiofile import readfile, writefile +#from pyaudio3dtools.audioarray import resample + + +def readfile( + filename: str, nchannels: int = 1, fs: int = 48000, outdtype="float" +) -> Tuple[np.ndarray, int]: + """Read audio file (.pcm or .wav) + + Parameters + ---------- + filename: str + Input file path + nchannels: Optional[int] + Number of input channels, required for .pcm otherwise default = 1 + fs: Optional[int] + Input sampling rate, required for .pcm input file, otherwise default = 48000 (Hz) + outdtype: Optional[int] + Data type of output array, python builtin or np.dtype + + Returns + ------- + x: np array + audio signal array + fs: int + signal sampling frequency + + """ + _, file_extension = os.path.splitext(os.path.basename(filename)) + + if file_extension == ".wav": + fs, data = wav.read(filename) + if data.dtype == np.int32: + data = np.interp( + data, + (np.iinfo(np.int32).min, np.iinfo(np.int32).max), + (np.iinfo(np.int16).min, np.iinfo(np.int16).max), + ) + elif data.dtype == np.float32: + data = np.interp( + data, + (-1, 1), + (np.iinfo(np.int16).min, np.iinfo(np.int16).max), + ) + x = np.array(data, dtype=outdtype) + file_len = x.shape[0] + if x.ndim == 1: + # force to be a mtx + x = np.reshape(x, (file_len, 1)) + elif file_extension == ".pcm" or file_extension == ".raw": + x = np.fromfile(filename, dtype=np.int16).astype(outdtype) + signal_len = len(x) // nchannels + x = x.reshape(signal_len, nchannels) + else: + raise ValueError("Wrong input format. Use wav or pcm") + + return x, fs + + +def writefile(filename: str, x: np.ndarray, fs: int = 48000) -> None: + """Write audio file (.pcm or .wav) + + Parameters + ---------- + filename: str + Output file path (.pcm or .wav) + x: np array + Numpy 2D array of dimension: number of samples x number of channels + fs: Optional[int] + Output sampling rate, required for .pcm input file, otherwise default = 48000 (Hz) + + Returns + ------- + None + + """ + _, file_extension = os.path.splitext(os.path.basename(filename)) + + clipped_samples = np.sum( + np.logical_or(x < np.iinfo(np.int16).min, x > np.iinfo(np.int16).max) + ) + if clipped_samples > 0: + warnings.warn(f" Warning: {clipped_samples} samples clipped") + x = np.clip(x, np.iinfo(np.int16).min, np.iinfo(np.int16).max) + + if file_extension == ".wav": + x = x.astype(np.int16) + wav.write(filename, fs, x) + elif file_extension == ".pcm" or file_extension == ".raw": + x = x.astype("int16").reshape(-1, 1) + x.tofile(filename) + else: + raise ValueError("Wrong input format. Use wav or pcm") + +def resample(x: np.ndarray, in_freq: int, out_freq: int) -> np.ndarray: + """Resample a multi-channel audio array + + Parameters + ---------- + x: numpy array + Input array + in_fs: int + Input sampling rate + out_fs: int + Output sampling rate + + Returns + ------- + y: + Output resampled numpy array + + """ + + if in_freq == out_freq or out_freq is None: + y = x + else: + # get gcd of original and deisred frequency + gcd = math.gcd(in_freq, out_freq) + + # calculate up-sampling factor + up_factor = int(out_freq / gcd) + + # calculate downsampling factor + down_factor = int(in_freq / gcd) + + # resample data using polyphase filtering across columns/channels + if x.ndim == 2: + y = sig.resample_poly(x[:, 0], up_factor, down_factor) + y = np.reshape(y, (y.shape[0], 1)) + for k in range(1, x.shape[1]): + a = sig.resample_poly(x[:, k], up_factor, down_factor) + a = np.reshape(a, (a.shape[0], 1)) + y = np.append(y, a, axis=1) + else: + y = sig.resample_poly(x, up_factor, down_factor) + + return y class MLDConformance: @@ -80,6 +219,7 @@ class MLDConformance: self.cut_build_path = args.cut_build_path self.filter = args.filter self.mldbin = os.path.join(self.toolsdir, platform.system(), "mld") + self.wavdiffbin = os.path.join(self.toolsdir, platform.system(), "wav-diff") self.CutBins = dict() self.mldcsv = dict() self.sampleStats = dict() @@ -488,6 +628,9 @@ class MLDConformance: mldFile = os.path.join( tmpdir, f"{tempfile.gettempprefix()}_ch{ch}_MLD.csv" ) + mldFile2 = os.path.join( + tmpdir, f"{tempfile.gettempprefix()}_ch{ch}_MLD2.txt" + ) refFileMono = os.path.join( tmpdir, os.path.basename(refFile).replace(".wav", f"_ch{ch}.wav") ) @@ -505,7 +648,26 @@ class MLDConformance: dutFileMono, ] self.process(" ".join(command)) - mldThisChan = np.loadtxt(mldFile, delimiter=" ", dtype=float) + + command = [ + self.wavdiffbin, + "-s", + refFileMono, + dutFileMono, + ] + with open(mldFile2, "w") as fd: + c = subprocess.run( + " ".join(command), stdout=fd, stderr=subprocess.STDOUT, text=True, shell=True + ) + if c.returncode: + with open(self.failedCmdsFile, "a") as f: + f.write(command + "\n") + self.failedTests.value += 1 + + mldThisChan = np.loadtxt(mldFile2, delimiter=";", dtype=float, skiprows=1) + mldThisChan = mldThisChan[:, 2] + fd.close() + mldThisChan2 = np.loadtxt(mldFile, delimiter=" ", dtype=float) if ch == 0: mldThisFile = mldThisChan else: -- GitLab From 02e78c2e0950a4aba1e3a1bd33bf9ad0ba87de3e Mon Sep 17 00:00:00 2001 From: Rishabh Tyagi Date: Wed, 5 Nov 2025 23:22:42 +1100 Subject: [PATCH 05/16] merge isar mr --- scripts/ivas_conformance/runConformance.py | 28 +++++++--------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 025a64df77..6cbf8d902c 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -724,9 +724,6 @@ class MLDConformance: for ch in range(nChans): mldFile = os.path.join( - tmpdir, f"{tempfile.gettempprefix()}_ch{ch}_MLD.csv" - ) - mldFile2 = os.path.join( tmpdir, f"{tempfile.gettempprefix()}_ch{ch}_MLD2.txt" ) refFileMono = os.path.join( @@ -737,15 +734,6 @@ class MLDConformance: ) writefile(refFileMono, refSamples[:, ch], 48000) writefile(dutFileMono, dutSamples[:, ch], 48000) - command = [ - self.mldbin, - "-o", - mldFile, - "-s", - refFileMono, - dutFileMono, - ] - self.process(" ".join(command)) command = [ self.wavdiffbin, @@ -753,19 +741,21 @@ class MLDConformance: refFileMono, dutFileMono, ] - with open(mldFile2, "w") as fd: + + with open(mldFile, "w") as fd: c = subprocess.run( " ".join(command), stdout=fd, stderr=subprocess.STDOUT, text=True, shell=True ) - if c.returncode: - with open(self.failedCmdsFile, "a") as f: - f.write(command + "\n") - self.failedTests.value += 1 - mldThisChan = np.loadtxt(mldFile2, delimiter=";", dtype=float, skiprows=1) + #if c.returncode: + # with open(self.failedCmdsFile, "a") as f: + # f.write(command + "\n") + # self.failedTests.value += 1 + + mldThisChan = np.loadtxt(mldFile, delimiter=";", dtype=float, skiprows=1) mldThisChan = mldThisChan[:, 2] fd.close() - mldThisChan2 = np.loadtxt(mldFile, delimiter=" ", dtype=float) + if ch == 0: mldThisFile = mldThisChan else: -- GitLab From c4deaf785175c3585d86de5c5f741f236b09ff2b Mon Sep 17 00:00:00 2001 From: rtyag Date: Thu, 6 Nov 2025 00:29:23 +1100 Subject: [PATCH 06/16] remove unused mld bin path --- scripts/ivas_conformance/runConformance.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 433861793d..747d13b252 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -221,7 +221,6 @@ class MLDConformance: exe_platform = platform.system() if platform.system() == 'Windows': exe_platform = 'Win32' - self.mldbin = os.path.join(self.toolsdir, platform.system(), "mld") self.wavdiffbin = os.path.join(self.toolsdir, platform.system(), "wav-diff") self.CutBins = dict() self.mldcsv = dict() -- GitLab From c8d6c172d8ae8a1131dad4b9231b2df1eb6d95b7 Mon Sep 17 00:00:00 2001 From: rtyag Date: Thu, 6 Nov 2025 00:33:23 +1100 Subject: [PATCH 07/16] add requirements.txt --- scripts/ivas_conformance/requirements.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 scripts/ivas_conformance/requirements.txt diff --git a/scripts/ivas_conformance/requirements.txt b/scripts/ivas_conformance/requirements.txt new file mode 100644 index 0000000000..7f20a7e547 --- /dev/null +++ b/scripts/ivas_conformance/requirements.txt @@ -0,0 +1,3 @@ +scipy>=1.5.2 +numpy>=1.19.2 + -- GitLab From 39a88ce61ab4db1aa677b66eace8138a4f442f40 Mon Sep 17 00:00:00 2001 From: rtyag Date: Thu, 6 Nov 2025 01:51:27 +1100 Subject: [PATCH 08/16] avoid reference to folders outside conformance package --- scripts/ivas_conformance/runConformance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 747d13b252..b19a81dd36 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -554,7 +554,7 @@ class MLDConformance: return " ".join([exec, *commands[1:]]) def reformatCommand(self, command: str, ref: bool = False) -> str: - command = command.replace("$TESTV_PATH", self.scriptsDir) + command = command.replace("$TESTV_PATH", self.testvecDir) ################ HACKS ######################### command = command.replace("_cut.192.fer", ".192") command = command.replace("_cut.192", ".192") -- GitLab From dfa517be56411b69ea5e9db3e5bca0e1a856182f Mon Sep 17 00:00:00 2001 From: rtyag Date: Thu, 6 Nov 2025 11:36:17 +1100 Subject: [PATCH 09/16] remove preapre short vectors from package generation --- ivas_be_conf_test_gen.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ivas_be_conf_test_gen.sh b/ivas_be_conf_test_gen.sh index 2536de1f89..e1810c6130 100644 --- a/ivas_be_conf_test_gen.sh +++ b/ivas_be_conf_test_gen.sh @@ -4,7 +4,6 @@ cp IVAS_cod IVAS_cod_ref cp IVAS_dec IVAS_dec_ref cp IVAS_rend IVAS_rend_ref cp ISAR_post_rend ISAR_post_rend_ref -python3 scripts/prepare_combined_format_inputs.py python3 -m pytest -q tests/codec_be_on_mr_nonselection tests/renderer_short/test_renderer.py tests/split_rendering/test_split_rendering.py -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt rm -rf testvec -- GitLab From eb5c8b5c58756d274932caee5f6d0e7035ad7e82 Mon Sep 17 00:00:00 2001 From: Ripinder Singh Date: Thu, 6 Nov 2025 14:58:49 +1100 Subject: [PATCH 10/16] Replace multiprocessing.Process with multiprocessing.Pool API * Limits the active process to number of CPUs * On linux this prevents too many open file error * Have to revert the progress stats reporting as shared variable usage in pool is very different vs process due to difference of fork() vs fixed processes working with messages. Signed-off-by: Ripinder Singh --- scripts/ivas_conformance/runConformance.py | 40 +++++----------------- 1 file changed, 8 insertions(+), 32 deletions(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index b19a81dd36..041a905d35 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -38,7 +38,7 @@ import subprocess import tempfile import sys from typing import Tuple -from multiprocessing import Process, Value +from multiprocessing import Pool import shutil import scipy.io.wavfile as wav import warnings @@ -255,8 +255,6 @@ class MLDConformance: self.testvecDir = args.testvecDir self.toolsdir = os.path.join(self.scriptsDir, "tools") self.testvDir = os.path.join(self.testvecDir, "testv") - self.executedTests = Value("i", 0) - self.failedTests = Value("i", 0) self.setup() def accumulateCommands(self): @@ -396,22 +394,15 @@ class MLDConformance: ) self.process(command=" ".join(refDecCmd)) - self.executedTests.value += 1 self.stats() def runReferenceGeneration(self, encTag="ENC"): - processes = list() # Multiprocess list commands = conformance.Commands[encTag] self.totalTests = len(commands) if not self.args.no_multi_processing: - for commandIdx, command in enumerate(commands): - p = Process( - target=self.genEncoderReferences, args=(command, commandIdx, encTag) - ) - processes.append(p) - p.start() - for p in processes: - p.join() + with Pool() as pool: + args = [(command, commandIdx, encTag) for commandIdx, command in enumerate(commands)] + pool.starmap(self.genEncoderReferences, args) else: for commandIdx, command in enumerate(commands): conformance.genEncoderReferences(command, commandIdx, encTag) @@ -631,18 +622,13 @@ class MLDConformance: self.runOneIsarDecoderTest(command) else: assert False, f"Un-implemented Tag {tag}" - self.executedTests.value += 1 self.stats() def runTag(self, tag: str): - self.executedTests.value = 0 - self.failedTests.value = 0 # reset MLD, Sample Stats open(self.mldcsv[tag], "w").close() with open(self.sampleStats[tag], "w") as f: f.write(f"PYTESTTAG, MAXDIFF, RMSdB, BEFRAMES_PERCENT, MAX_MLD\n") - - processes = list() # Multiprocess list commands = list() if self.filter: for command in self.Commands[tag]: @@ -656,15 +642,9 @@ class MLDConformance: f"Executing tests for {tag} {'Filter='+self.filter if self.filter else ''} ({self.totalTests} tests)" ) if not self.args.no_multi_processing: - for command in commands: - p = Process( - target=self.runOneCommand, - args=(tag, command), - ) - processes.append(p) - p.start() - for p in processes: - p.join() + with Pool() as pool: + args = [(tag, command) for command in commands] + pool.starmap(self.runOneCommand, args) else: for command in commands: self.runOneCommand(tag, command) @@ -683,15 +663,11 @@ class MLDConformance: if c.returncode: with open(self.failedCmdsFile, "a") as f: f.write(command + "\n") - self.failedTests.value += 1 # c.check_returncode() return 0 def stats(self): - print( - f"Executed: {self.executedTests.value} / {self.totalTests} Failed: {self.failedTests.value}", - end="\r", - ) + pass def getSampleStats(self, refSamples: np.ndarray, dutSamples: np.ndarray): nSamples = min(refSamples.shape[0], dutSamples.shape[0]) -- GitLab From 056dff50c844b49c5a84469e49c3ab2386690753 Mon Sep 17 00:00:00 2001 From: rtyag Date: Thu, 6 Nov 2025 23:23:47 +1100 Subject: [PATCH 11/16] workaround for failing rtp cases --- scripts/ivas_conformance/runConformance.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 041a905d35..5db9df31a7 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -383,7 +383,9 @@ class MLDConformance: command = self.reformatCommand( command=self.Commands[decTag][decCommandIdx], ref=True ) - command = command.replace("-VOIP", "") + + if "-VOIP_hf_only" not in command: + command = command.replace("-VOIP", "") if encTag == "ISAR_ENC": refDecCmd = [self.RefBins[decTag]] + command.split()[1:] else: @@ -435,7 +437,8 @@ class MLDConformance: dutDecOutputFile = dutEncOutput.replace(".192", "_CUT_REFDECODED.wav") decCommandIdx = self.EncoderToDecoderCmdMap[encCommandIdx] command = self.reformatCommand(command=self.Commands["DEC"][decCommandIdx]) - command = command.replace("-VOIP", "") + if "-VOIP_hf_only" not in command: + command = command.replace("-VOIP", "") dutDecCmd = ( [self.RefBins["DEC"]] + command.split()[1:-2] @@ -547,10 +550,10 @@ class MLDConformance: def reformatCommand(self, command: str, ref: bool = False) -> str: command = command.replace("$TESTV_PATH", self.testvecDir) ################ HACKS ######################### - command = command.replace("_cut.192.fer", ".192") - command = command.replace("_cut.192", ".192") - command = command.replace(".fer.192", ".192") - command = command.replace(".192.fer", ".192") + #command = command.replace("_cut.192.fer", ".192") + #command = command.replace("_cut.192", ".192") + #command = command.replace(".fer.192", ".192") + #command = command.replace(".192.fer", ".192") ################################################## if ref: @@ -761,6 +764,7 @@ class MLDConformance: if N == 0: continue m0 = np.sum(mdlValues == 0) + m05 = np.sum(mdlValues <= 0.5) m1 = np.sum(mdlValues <= 1.0) m2 = np.sum(mdlValues <= 2.0) m5 = np.sum(mdlValues <= 5.0) @@ -770,10 +774,11 @@ class MLDConformance: print(f"<{tag}> Total Frames: {N}") print(f"<{tag}> MAX MLD across all frames : {mdlValues.max()}") print(f"<{tag}> Frames with MLD == 0 : {m0} frames ({PCNT(m0)}%)") + print(f"<{tag}> Frames with MLD <= 0.5 : {m05} frames ({PCNT(m05)}%)") print(f"<{tag}> Frames with MLD <= 1 : {m1} frames ({PCNT(m1)}%)") print(f"<{tag}> Frames with MLD <= 2 : {m2} frames ({PCNT(m2)}%)") print(f"<{tag}> Frames with MLD <= 5 : {m5} frames ({PCNT(m5)}%)") - print(f"<{tag}> BE frames percentage = {bePercentAvg}") + print(f"<{tag}> BE samples percentage = {bePercentAvg}") print(f"<{tag}> max absolute diff = {maxDiffmax}, sample range (-32768, 32767)") print("##########################################################\n") -- GitLab From 1a67be0a9d7e87ff6f668d7af740e6680f7c16c6 Mon Sep 17 00:00:00 2001 From: rtyag Date: Fri, 7 Nov 2025 09:18:31 +1100 Subject: [PATCH 12/16] merge issue fix for windows --- scripts/ivas_conformance/runConformance.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 5db9df31a7..36f92b9d91 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -219,9 +219,9 @@ class MLDConformance: self.cut_build_path = args.cut_build_path self.filter = args.filter exe_platform = platform.system() - if platform.system() == 'Windows': + if exe_platform == 'Windows': exe_platform = 'Win32' - self.wavdiffbin = os.path.join(self.toolsdir, platform.system(), "wav-diff") + self.wavdiffbin = os.path.join(self.toolsdir, exe_platform, "wav-diff") self.CutBins = dict() self.mldcsv = dict() self.sampleStats = dict() @@ -728,16 +728,11 @@ class MLDConformance: c = subprocess.run( " ".join(command), stdout=fd, stderr=subprocess.STDOUT, text=True, shell=True ) - - #if c.returncode: - # with open(self.failedCmdsFile, "a") as f: - # f.write(command + "\n") - # self.failedTests.value += 1 - - mldThisChan = np.loadtxt(mldFile, delimiter=";", dtype=float, skiprows=1) - mldThisChan = mldThisChan[:, 2] fd.close() + mldThisChan = np.loadtxt(mldFile, delimiter=";", dtype=float, skiprows=1) + mldThisChan = mldThisChan[:, 2] + if ch == 0: mldThisFile = mldThisChan else: -- GitLab From 961ad4a7e3dea290996a4998f50cf3d6568be330 Mon Sep 17 00:00:00 2001 From: Ripinder Singh Date: Fri, 7 Nov 2025 12:38:29 +1100 Subject: [PATCH 13/16] Split DUT test execution and analysis to two different steps * DUT tests can be executed in pass 1 * Analysis of DUT outputs in pass 2 * Updates to Readme.md Signed-off-by: Ripinder Singh --- scripts/ivas_conformance/README.md | 286 ++++++++- scripts/ivas_conformance/runConformance.py | 693 ++++++++++++--------- 2 files changed, 656 insertions(+), 323 deletions(-) diff --git a/scripts/ivas_conformance/README.md b/scripts/ivas_conformance/README.md index e5fb0824ff..4638af195b 100644 --- a/scripts/ivas_conformance/README.md +++ b/scripts/ivas_conformance/README.md @@ -1,22 +1,278 @@ -# IVAS conformance scripts +# IVAS Conformance Scripts -This folder contains scripts for running IVAS conformance tests. This is a placeholder file for instructions. +This folder contains scripts for running IVAS conformance tests. -no-BE conformance USAGE +## Setup for Reference Platform + Reference platform is Ubuntu 24.04 -Following CMDs needs to be executed from ivas-codec root folder: +- Verify the Ubuntu Linux release is 24.04 -################generate testvec package and encoder refs (temporary step to obtain testvec package, this step will be removed in final delivery) ############ -sh ivas_be_conf_test_gen.sh -PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/ref_bin --cut_build_path=testvec/cut_bin --test-mode=ENC --regenerate-enc-refs + ```shell + lsb_release -d | grep Ubuntu + ``` + # It might be similar to Ubuntu 24.04.3 LTS +- Install Clang 18 compiler -Encoder conformance: - - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/ref_bin --cut_build_path=testvec/cut_bin --test-mode=ENC -Decoder conformance: - - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/ref_bin --cut_build_path=testvec/cut_bin --test-mode=DEC -Renderer conformance: - - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/ref_bin --cut_build_path=testvec/cut_bin --test-mode=REND -Split renderer confomance: - - + ```shell + sudo apt install clang-18 + clang-18 --version + ``` + + Example version observed on Ubuntu 24.04.3 LTS + + ```text + Ubuntu clang version 18.1.3 (1ubuntu1) + Target: x86_64-pc-linux-gnu + Thread model: posix + InstalledDir: /usr/bin + ``` + + It might be required to set Clang-18 as the defauly clang on the machine + + ```shell + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 100 + ``` + +- Install Python3.13 on Ubuntu 24.04 LTS + + ```shell + sudo apt install python3.13 python3.13-venv + ``` + +- Create virtual environment for Python 3.13 and install requirements + + ```shell + python3.13 -m venv pyConformance + source pyConformance/bin/activate + cd ivas-codec + python -m pip install -r tests/requirements.txt + ``` + +## Reference Conformance Package Generation + To generate reference conformance package for distribution + #### Generate Reference Outputs and Readme.txt files + + ```shell + sh ivas_be_conf_test_gen.sh + ``` + +
+ Example Output + 

+    ::::::::::::::::::::::::
+    ------------------------------------------
+    Generated html report: file:///home/dolby/git/ivas-codec/report_cmd.htm
+    ------------------------------------------
+    =================================================
+    2571 passed, 538 skipped, 230 xfailed in 377.10s (0:06:17)
+    =================================================
+    Identified 5430 files from scripts
+    Removed 1515 files
+    Kept 5422 files
+
+
+ +#### Generate Reference Decoded Outputs for the Reference Encoded files + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --regenerate-enc-refs + ``` + +#### Generate a conformance package zip + + ```shell + zip -r conformance.zip testvec scripts/ivas_conformance scripts/tools + ``` + +## Run CUT tests on Target platform + + To run CUT binaries on the targeted platform, it is necessary to replicate the initial setup for python and dependency packages. The CUT build of the IVAS binaries should be made available in a selected folder and needed for the next step + + To run IVAS DUT commands on the TARGET platform (may be different from ubuntu/linux) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR + ``` + +
+ Example Output of CUT execution + 

+    Accumulating commands from Readme_IVAS_dec.txt
+    Accumulating commands from Readme_IVAS_rend.txt
+    Accumulating commands from Readme_IVAS_enc.txt
+    Accumulating commands from Readme_IVAS_ISAR_post_rend.txt
+    Accumulating commands from Readme_IVAS_ISAR_dec.txt
+    Accumulating commands from Readme_IVAS_JBM_dec.txt
+    No of tests :
+        ENC : 381
+        DEC : 637
+        REND : 666
+        ISAR_ENC : 1032
+        ISAR : 1032
+    Executing tests for ENC   (381 tests)
+    Executing tests for DEC   (637 tests)
+    Executing tests for REND   (666 tests)
+    Executing tests for ISAR_ENC   (1032 tests)
+    Executing tests for ISAR   (1032 tests)
+
+
+ + This should generate outputs in scripts/CUT_OUTPUTS folder which looks like below:- + + ```shell + CUT_OUTPUTS + +- runlog.txt : Dump of all the commands run and the outputs (mostlyjumbled up due to multiprocessing) + +- failedCmds.txt : Log of all the shell commands that failed execution + +- dec/ : Folder containing all decoder tests CUT outputs + +- enc/ : Folder containing all encoder tests CUT outputs + +- renderer_short/ : Folder containing all renderer tests CUT outputs + +- split_rendering/ : Folder containing all split rendering enc/dec testsCUT outputs + ``` + +## Perform the MLD based analysis on the CUT outputs on refernce platform (Ubuntu 24.04) + + If CUT test execution is done on a different platform, the scripts/CUT_OUTPUTS must be copied and provided in the reference platform's scripts/CUT_OUTPUTS. Then the following command is used to perform MLD based analysis on the same, encoded outputs will be implicitly decoded using reference decoder executables and MLD analysis performed on the reference decoded outputs. + + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse + ``` + +
+Example Output of CUT Analysis +

+Accumulating commands from Readme_IVAS_dec.txt
+Accumulating commands from Readme_IVAS_rend.txt
+Accumulating commands from Readme_IVAS_enc.txt
+Accumulating commands from Readme_IVAS_ISAR_post_rend.txt
+Accumulating commands from Readme_IVAS_ISAR_dec.txt
+Accumulating commands from Readme_IVAS_JBM_dec.txt
+No of tests :
+    ENC : 381
+    DEC : 637
+    REND : 666
+    ISAR_ENC : 1032
+    ISAR : 1032
+Analysing tests for ENC   (381 tests)
+##########################################################
+<ENC> Total Frames: 2635800
+<ENC> MAX MLD across all frames : 0.0
+<ENC> Frames with MLD == 0 : 2635800 frames (100.0%)
+<ENC> Frames with MLD <= 1 : 2635800 frames (100.0%)
+<ENC> Frames with MLD <= 2 : 2635800 frames (100.0%)
+<ENC> Frames with MLD <= 5 : 2635800 frames (100.0%)
+<ENC> BE frames percentage = 100.0
+<ENC> max absolute diff = 0.0, sample range (-32768, 32767)
+##########################################################
+Analysing tests for DEC   (637 tests)
+##########################################################
+<DEC> Total Frames: 4342140
+<DEC> MAX MLD across all frames : 0.0
+<DEC> Frames with MLD == 0 : 4342140 frames (100.0%)
+<DEC> Frames with MLD <= 1 : 4342140 frames (100.0%)
+<DEC> Frames with MLD <= 2 : 4342140 frames (100.0%)
+<DEC> Frames with MLD <= 5 : 4342140 frames (100.0%)
+<DEC> BE frames percentage = 100.0
+<DEC> max absolute diff = 0.0, sample range (-32768, 32767)
+##########################################################
+Analysing tests for REND   (666 tests)
+##########################################################
+<REND> Total Frames: 4799952
+<REND> MAX MLD across all frames : 0.0
+<REND> Frames with MLD == 0 : 4799952 frames (100.0%)
+<REND> Frames with MLD <= 1 : 4799952 frames (100.0%)
+<REND> Frames with MLD <= 2 : 4799952 frames (100.0%)
+<REND> Frames with MLD <= 5 : 4799952 frames (100.0%)
+<REND> BE frames percentage = 100.0
+<REND> max absolute diff = 0.0, sample range (-32768, 32767)
+##########################################################
+Analysing tests for ISAR_ENC   (1032 tests)
+##########################################################
+<ISAR_ENC> Total Frames: 2125956
+<ISAR_ENC> MAX MLD across all frames : 0.0
+<ISAR_ENC> Frames with MLD == 0 : 2125956 frames (100.0%)
+<ISAR_ENC> Frames with MLD <= 1 : 2125956 frames (100.0%)
+<ISAR_ENC> Frames with MLD <= 2 : 2125956 frames (100.0%)
+<ISAR_ENC> Frames with MLD <= 5 : 2125956 frames (100.0%)
+<ISAR_ENC> BE frames percentage = 100.0
+<ISAR_ENC> max absolute diff = 0.0, sample range (-32768,32767)
+##########################################################
+Analysing tests for ISAR   (1032 tests)
+##########################################################
+<ISAR> Total Frames: 2125956
+<ISAR> MAX MLD across all frames : 0.0
+<ISAR> Frames with MLD == 0 : 2125956 frames (100.0%)
+<ISAR> Frames with MLD <= 1 : 2125956 frames (100.0%)
+<ISAR> Frames with MLD <= 2 : 2125956 frames (100.0%)
+<ISAR> Frames with MLD <= 5 : 2125956 frames (100.0%)
+<ISAR> BE frames percentage = 100.0
+<ISAR> max absolute diff = 0.0, sample range (-32768, 32767)
+##########################################################
+
+
+ +## Executing specific tests only + +All CUT tests can be run specifically for IVAS Encoder,IVAS Decoder,IVAS Renderer, ISAR Encoder and ISAR Decoder only. The commandline allows for ```-test-mode=``` for this functionality, examples : - + +- Run DUT IVAS Encoder Tests Only (on Target Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ENC + ``` + +- Analyse DUT IVAS Encoder Outputs Only (on Reference Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ENC --analyse + ``` + +- Run DUT IVAS Decoder Tests Only (on Target Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=DEC + ``` + +- Analyse DUT IVAS Decoder Outputs Only (on Reference Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=DEC --analyse + ``` + +- Run DUT IVAS Renderer Tests Only (on Target Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=REND + ``` + +- Analyse DUT Renderer Outputs Only (on Reference Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=REND --analyse + ``` + +- Run DUT ISAR Encoder Tests Only (on Target Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ISAR_ENC + ``` + +- Analyse DUT ISAR Encoder Outputs Only (on Reference Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ISAR_ENC --analyse + ``` + +- Run DUT ISAR Decoder Tests Only (on Target Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ISAR + ``` + +- Analyse DUT ISAR Decoder Outputs Only (on Reference Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ISAR --analyse + ``` diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 36f92b9d91..92596403df 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -39,6 +39,8 @@ import tempfile import sys from typing import Tuple from multiprocessing import Pool +from dataclasses import dataclass +from typing import Union import shutil import scipy.io.wavfile as wav import warnings @@ -138,6 +140,7 @@ def writefile(filename: str, x: np.ndarray, fs: int = 48000) -> None: else: raise ValueError("Wrong input format. Use wav or pcm") + def resample(x: np.ndarray, in_freq: int, out_freq: int) -> np.ndarray: """Resample a multi-channel audio array @@ -183,32 +186,51 @@ def resample(x: np.ndarray, in_freq: int, out_freq: int) -> np.ndarray: return y -class MLDConformance: - IVAS_Bins = { - "ENC": "IVAS_cod", - "DEC": "IVAS_dec", - "REND": "IVAS_rend", - "ISAR_ENC": "IVAS_dec", - "ISAR": "ISAR_post_rend", - } +IVAS_Bins = { + "ENC": "IVAS_cod", + "DEC": "IVAS_dec", + "REND": "IVAS_rend", + "ISAR_ENC": "IVAS_dec", + "ISAR": "ISAR_post_rend", +} + + +@dataclass +class TestDesciptor: + dutOutput: str = "" + refOutput: str = "" + rawCmdline: str = "" + dutCmdline: str = "" + refCmdline: str = "" + + +@dataclass +class BitstrmTestDescriptor(TestDesciptor): + rawDecCmdline: str = "" + dutDecCmdline: str = "" + refDecCmdline: str = "" + +class MLDConformance: def setupCommon(self): self.Commands = dict() + self.TestDesc = dict[ + str, dict[str, Union[TestDesciptor, BitstrmTestDescriptor]] + ]() self.EncoderToDecoderCmdMap = dict() self.IsarEncoderToDecoderCmdMap = dict() - for tag in MLDConformance.IVAS_Bins.keys(): + for tag in IVAS_Bins.keys(): self.Commands[tag] = list() # CREATE OUTPUT DIRECTORY STRUCTURE : CLEAN PREV OUTPUT self.outputDir = os.path.join(self.scriptsDir, "CUT_OUTPUTS") - if os.path.exists(self.outputDir): - if not self.args.dec_for_dut_enc: - shutil.rmtree(self.outputDir, ignore_errors=False) + if self.args.clean_output_dir and os.path.exists(self.outputDir): + shutil.rmtree(self.outputDir, ignore_errors=False) os.makedirs(self.outputDir, exist_ok=True) subdirs = ["enc", "dec", "renderer_short", "split_rendering"] for odir in subdirs: - os.makedirs(os.path.join(self.testvDir, "ref", odir), exist_ok=True) - os.makedirs(os.path.join(self.outputDir, "dut", odir), exist_ok=True) + os.makedirs(os.path.join(self.testvDir, odir), exist_ok=True) + os.makedirs(os.path.join(self.outputDir, odir), exist_ok=True) self.logFile = os.path.join(self.outputDir, "runlog.txt") self.failedCmdsFile = os.path.join(self.outputDir, "failedCmds.txt") @@ -219,17 +241,15 @@ class MLDConformance: self.cut_build_path = args.cut_build_path self.filter = args.filter exe_platform = platform.system() - if exe_platform == 'Windows': - exe_platform = 'Win32' + if exe_platform == "Windows": + exe_platform = "Win32" self.wavdiffbin = os.path.join(self.toolsdir, exe_platform, "wav-diff") self.CutBins = dict() self.mldcsv = dict() self.sampleStats = dict() - for tag in MLDConformance.IVAS_Bins.keys(): - self.CutBins[tag] = os.path.join( - self.cut_build_path, MLDConformance.IVAS_Bins[tag] - ) + for tag in IVAS_Bins.keys(): + self.CutBins[tag] = os.path.join(self.cut_build_path, IVAS_Bins[tag]) self.mldcsv[tag] = os.path.join(self.outputDir, f"mld_{tag}.csv") self.sampleStats[tag] = os.path.join( self.outputDir, f"sampleStats_{tag}.csv" @@ -238,16 +258,13 @@ class MLDConformance: def setupRef(self): self.RefBins = dict() self.ref_build_path = self.args.ref_build_path - for tag in MLDConformance.IVAS_Bins.keys(): - self.RefBins[tag] = os.path.join( - self.ref_build_path, MLDConformance.IVAS_Bins[tag] - ) + for tag in IVAS_Bins.keys(): + self.RefBins[tag] = os.path.join(self.ref_build_path, IVAS_Bins[tag]) def setup(self): self.setupCommon() self.setupRef() - if not self.args.regenerate_enc_refs: - self.setupDUT() + self.setupDUT() def __init__(self, args) -> None: self.args = args @@ -263,23 +280,23 @@ class MLDConformance: basename, ext = os.path.splitext(file_name) if ("Readme_IVAS_" in basename) and ext == ".txt": print(f"Accumulating commands from {file_name}") + isISAREnc = "IVAS_ISAR_dec" in file_name file = os.path.join(root, file_name) - self.parseCommandsFile(file) - self.mapEncoderToDecoderCommands() - self.mapIsarEncToDecCommands() - print("No of tests :") - for key in self.Commands.keys(): - print(f" {key} : {len(self.Commands[key])}") - - def parseCommandsFile(self, filePath): - isISAREnc = "IVAS_ISAR_dec" in filePath - with open(filePath) as fp: - for line in fp.readlines(): - m = re.search(r"^\$(CUT_.+_BIN) ", line) - if m: - tag = m.group(1).split("_")[1] if not isISAREnc else "ISAR_ENC" - if tag in self.Commands.keys(): - self.Commands[tag].append(line) + with open(file) as fp: + for line in fp.readlines(): + m = re.search(r"^\$(CUT_.+_BIN) ", line) + if m: + tag = ( + m.group(1).split("_")[1] + if not isISAREnc + else "ISAR_ENC" + ) + self.Commands[tag].append(line) + self.TestDesc = self.createTestDescriptors() + + def getRendPyTestTag(self, command: str) -> str: + refRendOutputFile = self.getRendOutputFile(command) + return os.path.basename(refRendOutputFile).split(".")[-2] def getPcmPytestTag(self, command: str) -> str: decInput = ( @@ -303,178 +320,232 @@ class MLDConformance: def getIsarEncPytestTag(self, command: str) -> str: return os.path.basename(command.split()[-1]).split(".")[-3] - def mapIsarEncToDecCommands(self): - decoderPyTestTags = dict() - encoderPyTestTags = dict() - for idx, command in enumerate(self.Commands["ISAR"]): - decoderPyTestTags[self.getIsarDecPytestTag(command)] = idx - for idx, command in enumerate(self.Commands["ISAR_ENC"]): - encoderPyTestTags[self.getIsarEncPytestTag(command)] = idx - - for encTag in encoderPyTestTags.keys(): - if encTag in decoderPyTestTags.keys(): - self.IsarEncoderToDecoderCmdMap[encoderPyTestTags[encTag]] = ( - decoderPyTestTags[encTag] + def createTestDescriptors( + self, + ) -> dict[str, dict[str, Union[TestDesciptor, BitstrmTestDescriptor]]]: + testDesciptor = dict[ + str, dict[str, Union[TestDesciptor, BitstrmTestDescriptor]] + ]() + for tag in IVAS_Bins.keys(): + testDesciptor[tag] = dict[ + str, Union[TestDesciptor, BitstrmTestDescriptor] + ]() + + IvasDecCmdMap = dict() + IsarDecCmdMap = dict() + for command in self.Commands["REND"]: + pyTestTag = self.getRendPyTestTag(command) + refCmdline = self.setCommandExec( + tag="REND", command=self.reformatCommand(command, ref=True), ref=True + ) + dutCmdline = self.setCommandExec( + tag="REND", command=self.reformatCommand(command) + ) + refOutput = self.getRendOutputFile(refCmdline) + dutOutput = self.getRendOutputFile(dutCmdline) + testDesciptor["REND"][pyTestTag] = TestDesciptor( + rawCmdline=command, + refOutput=refOutput, + dutOutput=dutOutput, + dutCmdline=dutCmdline, + refCmdline=refCmdline, + ) + for command in self.Commands["DEC"]: + pyTestTag = self.getPcmPytestTag(command) + refCmdline = self.setCommandExec( + tag="DEC", command=self.reformatCommand(command, ref=True), ref=True + ) + dutCmdline = self.setCommandExec( + tag="DEC", command=self.reformatCommand(command) + ) + refOutput = self.getOutputFile(refCmdline) + dutOutput = self.getOutputFile(dutCmdline) + testDesciptor["DEC"][pyTestTag] = TestDesciptor( + rawCmdline=command, + refOutput=refOutput, + dutOutput=dutOutput, + dutCmdline=dutCmdline, + refCmdline=refCmdline, + ) + IvasDecCmdMap[pyTestTag] = command + for command in self.Commands["ISAR"]: + pyTestTag = self.getIsarDecPytestTag(command) + refCmdline = self.setCommandExec( + tag="ISAR", command=self.reformatCommand(command, ref=True), ref=True + ) + dutCmdline = self.setCommandExec( + tag="ISAR", command=self.reformatCommand(command) + ) + refOutput = self.getRendOutputFile(refCmdline) + dutOutput = self.getRendOutputFile(dutCmdline) + testDesciptor["ISAR"][pyTestTag] = TestDesciptor( + rawCmdline=command, + refOutput=refOutput, + dutOutput=dutOutput, + dutCmdline=dutCmdline, + refCmdline=refCmdline, + ) + IsarDecCmdMap[pyTestTag] = command + for command in self.Commands["ENC"]: + pyTestTag = self.getEncPytestTag(command) + refCmdline = self.setCommandExec( + tag="ENC", command=self.reformatCommand(command, ref=True), ref=True + ) + dutCmdline = self.setCommandExec( + tag="ENC", command=self.reformatCommand(command) + ) + refOutput = self.getOutputFile(refCmdline) + dutOutput = self.getOutputFile(dutCmdline) + assert ".192" in dutOutput, "Output file not identified in dut" + assert ".192" in refOutput, "Output file not identified in ref" + if pyTestTag in IvasDecCmdMap.keys(): + rawDecCmdline = IvasDecCmdMap[pyTestTag] + refDecCmdline = self.setCommandExec( + tag="DEC", + command=self.reformatCommand(rawDecCmdline, ref=True), + ref=True, + ) + dutDecCmdline = self.setCommandExec( + tag="DEC", command=self.reformatCommand(rawDecCmdline) + ) + testDesciptor["ENC"][pyTestTag] = BitstrmTestDescriptor( + rawCmdline=command, + rawDecCmdline=rawDecCmdline, + refOutput=refOutput, + dutOutput=dutOutput, + dutCmdline=dutCmdline, + refCmdline=refCmdline, + dutDecCmdline=dutDecCmdline, + refDecCmdline=refDecCmdline, ) - if self.args.verbose: - print( - f"{encTag} {encoderPyTestTags[encTag]} -> {decoderPyTestTags[encTag]}" - ) - print(f"{self.Commands['ISAR_ENC'][encoderPyTestTags[encTag]]}") - print(f"{self.Commands['ISAR'][decoderPyTestTags[encTag]]}") else: - print(f"{encTag} not fount in decoder") - print( - f"Mapped decoder tests for {len(self.IsarEncoderToDecoderCmdMap)} encoder tests out of {len(self.Commands['ISAR_ENC'])} tests" - ) - assert len(self.IsarEncoderToDecoderCmdMap) == len( - self.Commands["ISAR_ENC"] - ), "Failed to Map Encoder Commands to Decoder Commands" - - def mapEncoderToDecoderCommands(self): - decoderPyTestTags = dict() - encoderPyTestTags = dict() - for idx, command in enumerate(self.Commands["DEC"]): - decoderPyTestTags[self.getPcmPytestTag(command)] = idx - for idx, command in enumerate(self.Commands["ENC"]): - encoderPyTestTags[self.getEncPytestTag(command)] = idx - - for encTag in encoderPyTestTags.keys(): - if encTag in decoderPyTestTags.keys(): - self.EncoderToDecoderCmdMap[encoderPyTestTags[encTag]] = ( - decoderPyTestTags[encTag] + print(f"{pyTestTag} not found in decoder") + for command in self.Commands["ISAR_ENC"]: + pyTestTag = self.getIsarEncPytestTag(command) + refCmdline = self.setCommandExec( + tag="ISAR_ENC", + command=self.reformatCommand(command, ref=True), + ref=True, + ) + dutCmdline = self.setCommandExec( + tag="ISAR_ENC", command=self.reformatCommand(command) + ) + refOutput = self.getOutputFile(refCmdline) + dutOutput = self.getOutputFile(dutCmdline) + if pyTestTag in IsarDecCmdMap.keys(): + rawDecCmdline = IsarDecCmdMap[pyTestTag] + refDecCmdline = self.setCommandExec( + tag="ISAR", + command=self.reformatCommand(rawDecCmdline, ref=True), + ref=True, + ) + dutDecCmdline = self.setCommandExec( + tag="ISAR", command=self.reformatCommand(rawDecCmdline) + ) + testDesciptor["ISAR_ENC"][pyTestTag] = BitstrmTestDescriptor( + rawCmdline=command, + rawDecCmdline=rawDecCmdline, + refOutput=refOutput, + dutOutput=dutOutput, + dutCmdline=dutCmdline, + refCmdline=refCmdline, + dutDecCmdline=dutDecCmdline, + refDecCmdline=refDecCmdline, ) - if self.args.verbose: - print( - f"{encTag} {encoderPyTestTags[encTag]} -> {decoderPyTestTags[encTag]}" - ) - print(f"{self.Commands['ENC'][encoderPyTestTags[encTag]]}") - print(f"{self.Commands['DEC'][decoderPyTestTags[encTag]]}") else: - print(f"{encTag} not fount in decoder") - print( - f"Mapped decoder tests for {len(self.EncoderToDecoderCmdMap)} encoder tests out of {len(self.Commands['ENC'])} tests" - ) - assert len(self.EncoderToDecoderCmdMap) == len( - self.Commands["ENC"] - ), "Failed to Map Encoder Commands to Decoder Commands" - - def genEncoderReferences( - self, command: str, encCommandIdx: int, encTag: str = "ENC" - ): - # RUN ENCODER COMMAND LINE WITH REFERENCE ENCODER - refCommand = self.reformatCommand(command, ref=True) - refEncOutput = self.getOutputFile(refCommand) - # if not os.path.exists(refEncOutput): - self.process( - command=self.setCommandExec(tag=encTag, command=refCommand, ref=True) - ) + print(f"{pyTestTag} not found in ISAR decoder") + print("No of tests :") + for tag in testDesciptor.keys(): + print(f" {tag} : {len(testDesciptor[tag])}") - # FIND CORRESPONDING DECODER COMMAND - if encTag == "ISAR_ENC": - decTag = "ISAR" - decCommandIdx = self.IsarEncoderToDecoderCmdMap[encCommandIdx] - refDecOutputFile = refEncOutput.replace(".splt.bit", ".splt.REFDECODED.wav") - else: - decTag = "DEC" - decCommandIdx = self.EncoderToDecoderCmdMap[encCommandIdx] - refDecOutputFile = refEncOutput.replace(".192", "_REFDECODED.wav") + return testDesciptor - command = self.reformatCommand( - command=self.Commands[decTag][decCommandIdx], ref=True - ) - - if "-VOIP_hf_only" not in command: - command = command.replace("-VOIP", "") - if encTag == "ISAR_ENC": - refDecCmd = [self.RefBins[decTag]] + command.split()[1:] - else: - refDecCmd = ( - [self.RefBins[decTag]] - + command.split()[1:-2] - + [refEncOutput, refDecOutputFile] - ) + def genEncoderReferences(self, tag: str, encPytestTag: str): + # RUN ENCODER'S OUTPUT DECODED WITH REF DECODER + testDesc = self.TestDesc[tag][encPytestTag] + assert isinstance( + testDesc, BitstrmTestDescriptor + ), f"Expected bitstream test descriptor for {tag}" - self.process(command=" ".join(refDecCmd)) + # Decode the encoded output with Reference IVAS decoder + if tag == "ENC": + refDecOutputFile = testDesc.refOutput.replace(".192", "_REFDECODED.wav") + refDecCmd = testDesc.refDecCmdline.split()[:-2] + [ + testDesc.refOutput, + refDecOutputFile, + ] + else: + refDecOutputFile = testDesc.refOutput.replace(".splt.bit", ".wav") + refDecCmd = testDesc.refDecCmdline.split() + for idx, cmd in enumerate(refDecCmd): + if cmd == "-o" and (idx + 1) < len(refDecCmd): + refDecCmd[idx + 1] = refDecOutputFile + break + refDecCmd = ["" if x == "-VOIP" else x for x in refDecCmd] + refDecCmd = " ".join(refDecCmd) + self.process(command=refDecCmd) self.stats() def runReferenceGeneration(self, encTag="ENC"): - commands = conformance.Commands[encTag] - self.totalTests = len(commands) + selectedTests = list(self.TestDesc[encTag].keys()) + self.totalTests = len(selectedTests) if not self.args.no_multi_processing: with Pool() as pool: - args = [(command, commandIdx, encTag) for commandIdx, command in enumerate(commands)] + args = [(encTag, pyTestsTag) for pyTestsTag in selectedTests] pool.starmap(self.genEncoderReferences, args) else: - for commandIdx, command in enumerate(commands): - conformance.genEncoderReferences(command, commandIdx, encTag) - - def runOneEncoderTest(self, command: str): - encPytestTag = self.getEncPytestTag(command) - refEncOutput = self.getOutputFile(command) - refEncOutput = refEncOutput.replace( - "$CUT_PATH/ref/param_file/enc/", - f"{self.testvecDir}/testv/ref/param_file/enc/", - ) - refEncOutput = refEncOutput.replace( - "$CUT_PATH/ref/sba_bs/pkt/", - f"{self.testvecDir}/testv/ref/sba_bs/pkt/", - ) - refDecOutputFile = refEncOutput.replace(".192", "_REFDECODED.wav") - - # Run CUT Encoder - encCommandIdx = self.Commands["ENC"].index(command) - command = self.reformatCommand(command) - command = self.setCommandExec(tag="ENC", command=command) - dutEncOutput = self.getOutputFile(command) - - if not self.args.dec_for_dut_enc: - self.process(command=command) - - assert ".192" in dutEncOutput, "Output file not identified" - - # Decode the encoded output with Reference decoder - dutDecOutputFile = dutEncOutput.replace(".192", "_CUT_REFDECODED.wav") - decCommandIdx = self.EncoderToDecoderCmdMap[encCommandIdx] - command = self.reformatCommand(command=self.Commands["DEC"][decCommandIdx]) - if "-VOIP_hf_only" not in command: - command = command.replace("-VOIP", "") - dutDecCmd = ( - [self.RefBins["DEC"]] - + command.split()[1:-2] - + [dutEncOutput, dutDecOutputFile] - ) - self.process(command=" ".join(dutDecCmd)) - self.mld( - "ENC", encPytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile - ) - - def runOneDecoderTest(self, tag: str, command: str): - dutPytestTag = self.getPcmPytestTag(command) - refInputFile = command.split()[-2].replace( - "$REF_PATH/ref", f"{self.testvDir}/ref" - ) - # refInputFile = refInputFile.replace("_cut.192.fer", ".192") - # refInputFile = refInputFile.replace(".fer.192", ".192").replace(".192.fer", ".192").replace("_cut.192.fer", ".192").replace("_cut.192", ".192") - refDecOutput = self.getOutputFile(command).replace( - "$CUT_PATH/ref", f"{self.testvDir}/ref" - ) - command = self.reformatCommand(command) - # command = command.replace("-VOIP", "") - dutDecOutputFile = self.getOutputFile(command) - dutDecCmd = ( - [self.CutBins["DEC"]] - + command.split()[1:-2] - + [refInputFile, dutDecOutputFile] - ) - self.process(command=" ".join(dutDecCmd)) - + for pyTestsTag in selectedTests: + self.genEncoderReferences(encTag, pyTestsTag) + + def analyseWavOutputTest(self, tag: str, dutPytestTag: str): + testDesc = self.TestDesc[tag][dutPytestTag] + assert isinstance( + testDesc, TestDesciptor + ), f"Expected pcm test descriptor for {tag}" ##### skip MLD verification for files with only 1 frame as MLD does not run with such files. Possible solution: append 0s and then compare ##### - if refInputFile.find("_cut.192.fer") == -1: + if testDesc.rawCmdline.find("_cut.192.fer") == -1: self.mld( - "DEC", dutPytestTag, refFile=refDecOutput, dutFile=dutDecOutputFile + tag, + dutPytestTag, + refFile=testDesc.refOutput, + dutFile=testDesc.dutOutput, ) + def analyseOneEncoderTest(self, tag: str, encPytestTag: str): + testDesc = self.TestDesc[tag][encPytestTag] + assert isinstance( + testDesc, BitstrmTestDescriptor + ), f"Expected bitstream test descriptor for {tag}" + refDecOutputFile = testDesc.refOutput.replace(".192", "_REFDECODED.wav") + + dutDecOutputFile = testDesc.dutOutput.replace(".192", "_CUT_REFDECODED.wav") + # Decode the encoded output with Reference IVAS decoder + dutDecCmd = testDesc.refDecCmdline.split()[:-2] + [ + testDesc.dutOutput, + dutDecOutputFile, + ] + dutDecCmd = ["" if x == "-VOIP" else x for x in dutDecCmd] + dutDecCmd = " ".join(dutDecCmd) + self.process(command=dutDecCmd) + self.mld(tag, encPytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile) + + def analyseOneIsarEncoderTest(self, tag: str, pytestTag: str): + testDesc = self.TestDesc[tag][pytestTag] + assert isinstance( + testDesc, BitstrmTestDescriptor + ), f"Expected bitstream test descriptor for {tag}" + refDecOutputFile = testDesc.refOutput.replace(".splt.bit", ".wav") + dutDecOutputFile = testDesc.dutOutput.replace(".splt.bit", ".wav") + # Decode the encoded output with Reference ISAR decoder + dutDecCmd = testDesc.refDecCmdline.split() + for idx, cmd in enumerate(dutDecCmd): + if cmd == "-o" and (idx + 1) < len(dutDecCmd): + dutDecCmd[idx + 1] = dutDecOutputFile + if cmd == "-i" and (idx + 1) < len(dutDecCmd): + dutDecCmd[idx + 1] = testDesc.dutOutput + self.process(command=" ".join(dutDecCmd)) + self.mld(tag, pytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile) + def getRendOutputFile(self, command: str): cmds = command.split() for idx, cmd in enumerate(cmds): @@ -482,63 +553,6 @@ class MLDConformance: return cmds[idx + 1] assert False, "Outputname not found" - def runOneRendererTest(self, tag: str, command: str): - refRendOutputFile = self.getRendOutputFile(command).replace( - "$CUT_PATH/renderer_short", f"{self.testvDir}/renderer_short" - ) - rendPytestTag = os.path.basename(refRendOutputFile).split(".")[-2] - command = self.reformatCommand(command) - dutRendCmd = " ".join([self.CutBins["REND"]] + command.split()[1:]) - dutRendOutputFile = self.getRendOutputFile(dutRendCmd) - self.process(command=dutRendCmd) - self.mld( - "REND", rendPytestTag, refFile=refRendOutputFile, dutFile=dutRendOutputFile - ) - - def runOneIsarEncoderTest(self, command: str): - encCommandIdx = self.Commands["ISAR_ENC"].index(command) - decCommandIdx = self.IsarEncoderToDecoderCmdMap[encCommandIdx] - - isarEncPytestTag = self.getIsarEncPytestTag(command) - refEncCommand = self.reformatCommand(command, ref=True) - refEncOutput = self.getOutputFile(refEncCommand) - refDecOutputFile = refEncOutput.replace(".splt.bit", ".wav") - - # Run CUT Encoder - dutEncCommand = self.reformatCommand(command) - dutEncOutput = self.getOutputFile(dutEncCommand) - self.process(command=self.setCommandExec(tag="ISAR_ENC", command=dutEncCommand)) - - # Decode the encoded output with Reference decoder - dutDecCommand = self.reformatCommand( - command=self.Commands["ISAR"][decCommandIdx] - ) - dutDecOutputFile = self.getRendOutputFile(dutDecCommand) - self.process( - command=self.setCommandExec(tag="ISAR", command=dutDecCommand, ref=True) - ) - self.mld( - "ISAR_ENC", - isarEncPytestTag, - refFile=refDecOutputFile, - dutFile=dutDecOutputFile, - ) - - def runOneIsarDecoderTest(self, command: str): - isarEncPytestTag = self.getIsarDecPytestTag(command) - refDecCommand = self.reformatCommand(command, ref=True) - refDecOutputFile = self.getRendOutputFile(refDecCommand) - - # Decode the encoded output with Reference decoder - dutDecCommand = self.reformatCommand(command) - dutDecOutputFile = self.getRendOutputFile(dutDecCommand) - self.process( - command=self.setCommandExec(tag="ISAR", command=dutDecCommand) - ) - self.mld( - "ISAR", isarEncPytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile - ) - def getOutputFile(self, command: str): return command.split()[-1] @@ -549,13 +563,6 @@ class MLDConformance: def reformatCommand(self, command: str, ref: bool = False) -> str: command = command.replace("$TESTV_PATH", self.testvecDir) - ################ HACKS ######################### - #command = command.replace("_cut.192.fer", ".192") - #command = command.replace("_cut.192", ".192") - #command = command.replace(".fer.192", ".192") - #command = command.replace(".192.fer", ".192") - ################################################## - if ref: command = command.replace( "$CUT_PATH/dut/sba_bs/pkt/", f"{self.testvDir}/ref/sba_bs/pkt/" @@ -567,38 +574,38 @@ class MLDConformance: "$CUT_PATH/ref/param_file/", f"{self.testvDir}/ref/param_file/" ) command = command.replace( - "$CUT_PATH/renderer_short/ref/", f"{self.testvDir}/ref/renderer_short/" + "$CUT_PATH/renderer_short/ref/", f"{self.testvDir}/renderer_short/ref/" ) command = command.replace( "$CUT_PATH/split_rendering/ref", - f"{self.testvDir}/ref/split_rendering", + f"{self.testvDir}/split_rendering/ref", ) command = command.replace( "$CUT_PATH/ref/sba_bs/", f"{self.testvDir}/ref/sba_bs/" ) else: command = command.replace( - "$CUT_PATH/ref/param_file/enc/", f"{self.outputDir}/dut/enc/" + "$CUT_PATH/ref/param_file/enc/", f"{self.outputDir}/enc/" ) command = command.replace( - "$CUT_PATH/ref/param_file/dec/", f"{self.outputDir}/dut/dec/" + "$CUT_PATH/ref/param_file/dec/", f"{self.outputDir}/dec/" ) command = command.replace( - "$CUT_PATH/renderer_short/ref/", f"{self.outputDir}/dut/renderer_short/" + "$CUT_PATH/renderer_short/ref/", f"{self.outputDir}/renderer_short/ref" ) command = command.replace( "$CUT_PATH/split_rendering/cut/", - f"{self.outputDir}/dut/split_rendering/", + f"{self.outputDir}/split_rendering/cut/", ) command = command.replace( "$CUT_PATH/split_rendering/ref", - f"{self.outputDir}/dut/split_rendering", + f"{self.outputDir}/split_rendering/", ) command = command.replace( - "$CUT_PATH/ref/sba_bs/pkt/", f"{self.outputDir}/dut/enc/" + "$CUT_PATH/ref/sba_bs/pkt/", f"{self.outputDir}/enc/" ) command = command.replace( - "$CUT_PATH/ref/sba_bs/raw/", f"{self.outputDir}/dut/dec/" + "$CUT_PATH/ref/sba_bs/raw/", f"{self.outputDir}/dec/" ) command = command.replace( @@ -612,45 +619,73 @@ class MLDConformance: ) return command - def runOneCommand(self, tag: str, command: str): + def runOneCommand(self, tag: str, pyTestsTag: str): + # Run CUT Cmdline + testDesc = self.TestDesc[tag][pyTestsTag] + self.process(command=testDesc.dutCmdline) + self.stats() + + def analyseOneCommand(self, tag: str, pyTestsTag: str): if tag == "ENC": - self.runOneEncoderTest(command) + self.analyseOneEncoderTest(tag, pyTestsTag) elif tag == "DEC": - self.runOneDecoderTest(tag, command) + self.analyseWavOutputTest(tag, pyTestsTag) elif tag == "REND": - self.runOneRendererTest(tag, command) + self.analyseWavOutputTest(tag, pyTestsTag) elif tag == "ISAR_ENC": - self.runOneIsarEncoderTest(command) + self.analyseOneIsarEncoderTest(tag, pyTestsTag) elif tag == "ISAR": - self.runOneIsarDecoderTest(command) + self.analyseWavOutputTest(tag, pyTestsTag) else: assert False, f"Un-implemented Tag {tag}" self.stats() def runTag(self, tag: str): + selectedTests = list() + if self.filter: + for pyTestsTag in self.TestDesc[tag].keys(): + if self.filter in self.TestDesc[tag][pyTestsTag].rawCmdline: + selectedTests.append(pyTestsTag) + else: + selectedTests = list(self.TestDesc[tag].keys()) + + self.totalTests = len(selectedTests) + print( + f"Executing tests for {tag} {'Filter='+self.filter if self.filter else ''} ({self.totalTests} tests)" + ) + if not self.args.no_multi_processing: + with Pool() as pool: + args = [(tag, pyTestsTag) for pyTestsTag in selectedTests] + pool.starmap(self.runOneCommand, args) + else: + for pyTestsTag in selectedTests: + self.runOneCommand(tag, pyTestsTag) + + def analyseTag(self, tag: str): # reset MLD, Sample Stats open(self.mldcsv[tag], "w").close() with open(self.sampleStats[tag], "w") as f: f.write(f"PYTESTTAG, MAXDIFF, RMSdB, BEFRAMES_PERCENT, MAX_MLD\n") - commands = list() + selectedTests = list() if self.filter: - for command in self.Commands[tag]: - if self.filter in command: - commands.append(command) + for pyTestsTag in self.TestDesc[tag].keys(): + if self.filter in self.TestDesc[tag][pyTestsTag].rawCmdline: + selectedTests.append(pyTestsTag) else: - commands = self.Commands[tag] + selectedTests = list(self.TestDesc[tag].keys()) - self.totalTests = len(commands) + self.totalTests = len(selectedTests) print( - f"Executing tests for {tag} {'Filter='+self.filter if self.filter else ''} ({self.totalTests} tests)" + f"Analysing tests for {tag} {'Filter='+self.filter if self.filter else ''} ({self.totalTests} tests)" ) if not self.args.no_multi_processing: with Pool() as pool: - args = [(tag, command) for command in commands] - pool.starmap(self.runOneCommand, args) + args = [(tag, pyTestsTag) for pyTestsTag in selectedTests] + pool.starmap(self.analyseOneCommand, args) else: - for command in commands: - self.runOneCommand(tag, command) + for pyTestsTag in selectedTests: + self.analyseOneCommand(tag, pyTestsTag) + self.doAnalysis(selectTag=tag) def process(self, command) -> int: if self.args.verbose: @@ -687,6 +722,13 @@ class MLDConformance: def mld(self, tag, pytestTag, refFile, dutFile): mldThisFile = np.zeros(0) + with open(self.failedCmdsFile, "a") as f: + if not os.path.exists(refFile): + f.write(f"File does not exists: {refFile}\n") + return + if not os.path.exists(dutFile): + f.write(f"File does not exists : {dutFile}\n") + return with tempfile.TemporaryDirectory() as tmpdir: refSamples, fsR = readfile(refFile, outdtype="float") dutSamples, fsD = readfile(dutFile, outdtype="float") @@ -707,12 +749,14 @@ class MLDConformance: for ch in range(nChans): mldFile = os.path.join( tmpdir, f"{tempfile.gettempprefix()}_ch{ch}_MLD2.txt" - ) + ) refFileMono = os.path.join( - tmpdir, os.path.basename(refFile).replace(".wav", f"_REF_ch{ch}.wav") + tmpdir, + os.path.basename(refFile).replace(".wav", f"_REF_ch{ch}.wav"), ) dutFileMono = os.path.join( - tmpdir, os.path.basename(dutFile).replace(".wav", f"_DUT_ch{ch}.wav") + tmpdir, + os.path.basename(dutFile).replace(".wav", f"_DUT_ch{ch}.wav"), ) writefile(refFileMono, refSamples[:, ch], 48000) writefile(dutFileMono, dutSamples[:, ch], 48000) @@ -724,15 +768,20 @@ class MLDConformance: dutFileMono, ] + # Output on stdout with wavdiffbin with open(mldFile, "w") as fd: c = subprocess.run( - " ".join(command), stdout=fd, stderr=subprocess.STDOUT, text=True, shell=True + " ".join(command), + stdout=fd, + stderr=subprocess.STDOUT, + text=True, + shell=True, ) - fd.close() - mldThisChan = np.loadtxt(mldFile, delimiter=";", dtype=float, skiprows=1) + mldThisChan = np.loadtxt( + mldFile, delimiter=";", dtype=float, skiprows=1 + ) mldThisChan = mldThisChan[:, 2] - if ch == 0: mldThisFile = mldThisChan else: @@ -747,14 +796,26 @@ class MLDConformance: ) def doAnalysis(self, selectTag="all"): - keys = MLDConformance.IVAS_Bins.keys() if selectTag == "all" else [selectTag] + keys = IVAS_Bins.keys() if selectTag == "all" else [selectTag] for tag in keys: if os.path.exists(self.mldcsv[tag]): mdlValues = np.loadtxt(self.mldcsv[tag], delimiter=" ", dtype=float) - bePercent = np.loadtxt(self.sampleStats[tag], delimiter=",", dtype=float, skiprows=1, usecols=3) - maxDiff = np.loadtxt(self.sampleStats[tag], delimiter=",", dtype=float, skiprows=1, usecols=1) + bePercent = np.loadtxt( + self.sampleStats[tag], + delimiter=",", + dtype=float, + skiprows=1, + usecols=3, + ) + maxDiff = np.loadtxt( + self.sampleStats[tag], + delimiter=",", + dtype=float, + skiprows=1, + usecols=1, + ) bePercentAvg = np.average(bePercent) - maxDiffmax = np.max(maxDiff)*32768.0 + maxDiffmax = np.max(maxDiff) * 32768.0 N = mdlValues.shape[0] if N == 0: continue @@ -774,7 +835,9 @@ class MLDConformance: print(f"<{tag}> Frames with MLD <= 2 : {m2} frames ({PCNT(m2)}%)") print(f"<{tag}> Frames with MLD <= 5 : {m5} frames ({PCNT(m5)}%)") print(f"<{tag}> BE samples percentage = {bePercentAvg}") - print(f"<{tag}> max absolute diff = {maxDiffmax}, sample range (-32768, 32767)") + print( + f"<{tag}> max absolute diff = {maxDiffmax}, sample range (-32768, 32767)" + ) print("##########################################################\n") @@ -792,11 +855,13 @@ if __name__ == "__main__": parser.add_argument( "--ref_build_path", type=str, + default="", help="Path to the reference build folder containing IVAS Encoder, Decoder, Renderer and Post Render binaries", ) parser.add_argument( "--cut_build_path", type=str, + default="", help="Path to the CUT build folder containing IVAS Encoder, Decoder, Renderer and Post Render binaries", ) parser.add_argument( @@ -810,7 +875,7 @@ if __name__ == "__main__": default=False, action="store_true", help="process dut ENC files with REF decoder", - ) + ) parser.add_argument( "--verbose", default=False, @@ -834,7 +899,7 @@ if __name__ == "__main__": "--test-mode", type=str, default="ALL", - help='Choose tests to run ["ENC", "DEC", "REND", "ISAR", "ALL"]', + help='Choose tests to run ["ENC", "DEC", "REND", "ISAR", "ISAR_ENC", "ALL"]', ) parser.add_argument( "--no-multi-processing", @@ -843,16 +908,27 @@ if __name__ == "__main__": help="Disable multi-processing for sequential test run (debugging)", ) parser.add_argument( - "--analyse-only", + "--analyse", + default=False, + action="store_true", + help="Perform MLD analysis on CUT outputs generated", + ) + parser.add_argument( + "--report-only", + default=False, + action="store_true", + help="Do not run DUT, use existing mld and bitdiff stats files to generate analysis only", + ) + parser.add_argument( + "-c", + "--clean-output-dir", default=False, action="store_true", help="Do not run DUT, use existing mld and bitdiff stats files to generate analysis only", ) - args = parser.parse_args() conformance = MLDConformance(args) - conformance.accumulateCommands() if args.regenerate_enc_refs: @@ -860,10 +936,11 @@ if __name__ == "__main__": conformance.runReferenceGeneration(encTag="ENC") sys.exit(0) - testTags = ( - MLDConformance.IVAS_Bins.keys() if args.test_mode == "ALL" else [args.test_mode] - ) + testTags = IVAS_Bins.keys() if args.test_mode == "ALL" else [args.test_mode] for tag in testTags: - if not args.analyse_only: + if args.report_only: + conformance.doAnalysis(selectTag=tag) + elif not args.analyse: conformance.runTag(tag) - conformance.doAnalysis(selectTag=tag) + else: + conformance.analyseTag(tag) -- GitLab From e18902453ffb776e4ea008df7600930094b02c11 Mon Sep 17 00:00:00 2001 From: Ripinder Singh Date: Fri, 7 Nov 2025 14:23:00 +1100 Subject: [PATCH 14/16] Make refrence generation section collapsible * Fix markdown lint fixes Signed-off-by: Ripinder Singh --- scripts/ivas_conformance/README.md | 112 +++++++++++++++-------------- 1 file changed, 60 insertions(+), 52 deletions(-) diff --git a/scripts/ivas_conformance/README.md b/scripts/ivas_conformance/README.md index 4638af195b..78c006fc2d 100644 --- a/scripts/ivas_conformance/README.md +++ b/scripts/ivas_conformance/README.md @@ -3,6 +3,7 @@ This folder contains scripts for running IVAS conformance tests. ## Setup for Reference Platform + Reference platform is Ubuntu 24.04 - Verify the Ubuntu Linux release is 24.04 @@ -11,7 +12,7 @@ This folder contains scripts for running IVAS conformance tests. lsb_release -d | grep Ubuntu ``` - # It might be similar to Ubuntu 24.04.3 LTS + # It might be similar to Ubuntu 24.04.3 LTS - Install Clang 18 compiler @@ -29,7 +30,7 @@ This folder contains scripts for running IVAS conformance tests. InstalledDir: /usr/bin ``` - It might be required to set Clang-18 as the defauly clang on the machine + It might be required to set Clang-18 as the default clang on the machine ```shell sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 100 @@ -51,41 +52,48 @@ This folder contains scripts for running IVAS conformance tests. ``` ## Reference Conformance Package Generation - To generate reference conformance package for distribution - #### Generate Reference Outputs and Readme.txt files + +
+ Expand for detailed procedure + +To generate reference conformance package for distribution + +### Generate Reference Outputs and Readme.txt files ```shell sh ivas_be_conf_test_gen.sh ``` -
- Example Output - 

-    ::::::::::::::::::::::::
-    ------------------------------------------
-    Generated html report: file:///home/dolby/git/ivas-codec/report_cmd.htm
-    ------------------------------------------
-    =================================================
-    2571 passed, 538 skipped, 230 xfailed in 377.10s (0:06:17)
-    =================================================
-    Identified 5430 files from scripts
-    Removed 1515 files
-    Kept 5422 files
-
-
- -#### Generate Reference Decoded Outputs for the Reference Encoded files +
+ Example output of reference test generation +

+::::::::::::::::::::::::
+------------------------------------------
+Generated html report: file:///home/dolby/git/ivas-codec/report_cmd.htm
+------------------------------------------
+=================================================
+2571 passed, 538 skipped, 230 xfailed in 377.10s (0:06:17)
+=================================================
+Identified 5430 files from scripts
+Removed 1515 files
+Kept 5422 files
+
+
+ +### Generate Reference Decoded Outputs for the Reference Encoded files ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --regenerate-enc-refs ``` -#### Generate a conformance package zip +### Generate a conformance package zip ```shell zip -r conformance.zip testvec scripts/ivas_conformance scripts/tools ``` +
+ ## Run CUT tests on Target platform To run CUT binaries on the targeted platform, it is necessary to replicate the initial setup for python and dependency packages. The CUT build of the IVAS binaries should be made available in a selected folder and needed for the next step @@ -96,42 +104,42 @@ This folder contains scripts for running IVAS conformance tests. PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR ``` -
- Example Output of CUT execution - 

-    Accumulating commands from Readme_IVAS_dec.txt
-    Accumulating commands from Readme_IVAS_rend.txt
-    Accumulating commands from Readme_IVAS_enc.txt
-    Accumulating commands from Readme_IVAS_ISAR_post_rend.txt
-    Accumulating commands from Readme_IVAS_ISAR_dec.txt
-    Accumulating commands from Readme_IVAS_JBM_dec.txt
-    No of tests :
-        ENC : 381
-        DEC : 637
-        REND : 666
-        ISAR_ENC : 1032
-        ISAR : 1032
-    Executing tests for ENC   (381 tests)
-    Executing tests for DEC   (637 tests)
-    Executing tests for REND   (666 tests)
-    Executing tests for ISAR_ENC   (1032 tests)
-    Executing tests for ISAR   (1032 tests)
-
-
+
+Example Output of CUT execution +

+Accumulating commands from Readme_IVAS_dec.txt
+Accumulating commands from Readme_IVAS_rend.txt
+Accumulating commands from Readme_IVAS_enc.txt
+Accumulating commands from Readme_IVAS_ISAR_post_rend.txt
+Accumulating commands from Readme_IVAS_ISAR_dec.txt
+Accumulating commands from Readme_IVAS_JBM_dec.txt
+No of tests :
+    ENC : 381
+    DEC : 637
+    REND : 666
+    ISAR_ENC : 1032
+    ISAR : 1032
+Executing tests for ENC   (381 tests)
+Executing tests for DEC   (637 tests)
+Executing tests for REND   (666 tests)
+Executing tests for ISAR_ENC   (1032 tests)
+Executing tests for ISAR   (1032 tests)
+
+
This should generate outputs in scripts/CUT_OUTPUTS folder which looks like below:- ```shell - CUT_OUTPUTS - +- runlog.txt : Dump of all the commands run and the outputs (mostlyjumbled up due to multiprocessing) - +- failedCmds.txt : Log of all the shell commands that failed execution - +- dec/ : Folder containing all decoder tests CUT outputs - +- enc/ : Folder containing all encoder tests CUT outputs - +- renderer_short/ : Folder containing all renderer tests CUT outputs - +- split_rendering/ : Folder containing all split rendering enc/dec testsCUT outputs + CUT_OUTPUTS + +- runlog.txt : Dump of all the commands run and the outputs (mostly jumbled up due to multiprocessing) + +- failedCmds.txt : Log of all the shell commands that failed execution + +- dec/ : Folder containing all decoder tests CUT outputs + +- enc/ : Folder containing all encoder tests CUT outputs + +- renderer_short/ : Folder containing all renderer tests CUT outputs + +- split_rendering/ : Folder containing all split rendering enc/dec tests ``` -## Perform the MLD based analysis on the CUT outputs on refernce platform (Ubuntu 24.04) +## Perform the MLD based analysis on the CUT outputs on reference platform (Ubuntu 24.04) If CUT test execution is done on a different platform, the scripts/CUT_OUTPUTS must be copied and provided in the reference platform's scripts/CUT_OUTPUTS. Then the following command is used to perform MLD based analysis on the same, encoded outputs will be implicitly decoded using reference decoder executables and MLD analysis performed on the reference decoded outputs. -- GitLab From 303290fda6f50adbe12732ac06a5a6afb435438d Mon Sep 17 00:00:00 2001 From: rtyag Date: Fri, 7 Nov 2025 23:22:45 +1100 Subject: [PATCH 15/16] move ivas_be_conf_test_gen.sh to ivas_conformance folder --- scripts/ivas_conformance/README.md | 2 +- .../ivas_conformance/ivas_be_conf_test_gen.sh | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename ivas_be_conf_test_gen.sh => scripts/ivas_conformance/ivas_be_conf_test_gen.sh (100%) diff --git a/scripts/ivas_conformance/README.md b/scripts/ivas_conformance/README.md index 78c006fc2d..6355ecfcb6 100644 --- a/scripts/ivas_conformance/README.md +++ b/scripts/ivas_conformance/README.md @@ -61,7 +61,7 @@ To generate reference conformance package for distribution ### Generate Reference Outputs and Readme.txt files ```shell - sh ivas_be_conf_test_gen.sh + sh scripts/ivas_conformance/ivas_be_conf_test_gen.sh ```
diff --git a/ivas_be_conf_test_gen.sh b/scripts/ivas_conformance/ivas_be_conf_test_gen.sh similarity index 100% rename from ivas_be_conf_test_gen.sh rename to scripts/ivas_conformance/ivas_be_conf_test_gen.sh -- GitLab From c9802041a7601fc20c28a88fc5ce0ce8f702b9de Mon Sep 17 00:00:00 2001 From: Rishabh Tyagi Date: Mon, 10 Nov 2025 19:43:15 +1100 Subject: [PATCH 16/16] add BE conformance to runconformance script --- scripts/ivas_conformance/README.md | 46 +++++-- scripts/ivas_conformance/runConformance.py | 142 ++++++++++++++++----- 2 files changed, 146 insertions(+), 42 deletions(-) diff --git a/scripts/ivas_conformance/README.md b/scripts/ivas_conformance/README.md index 6355ecfcb6..0f617246b1 100644 --- a/scripts/ivas_conformance/README.md +++ b/scripts/ivas_conformance/README.md @@ -231,7 +231,13 @@ All CUT tests can be run specifically for IVAS Encoder,IVAS Decoder,IVAS Rendere PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ENC ``` -- Analyse DUT IVAS Encoder Outputs Only (on Reference Platform) +- Analyse BE conformance for DUT IVAS Encoder Outputs Only (on Reference Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ENC --analyse --be-test + ``` + +- Analyse NON-BE conformance for DUT IVAS Encoder Outputs Only (on Reference Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ENC --analyse @@ -243,10 +249,16 @@ All CUT tests can be run specifically for IVAS Encoder,IVAS Decoder,IVAS Rendere PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=DEC ``` -- Analyse DUT IVAS Decoder Outputs Only (on Reference Platform) +- Analyse BE conformance for DUT IVAS Decoder Outputs Only + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=DEC --analyse --be-test + ``` + +- Analyse NON-BE conformance DUT IVAS Decoder Outputs Only (on Reference Platform) ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=DEC --analyse + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=DEC --analyse ``` - Run DUT IVAS Renderer Tests Only (on Target Platform) @@ -255,10 +267,16 @@ All CUT tests can be run specifically for IVAS Encoder,IVAS Decoder,IVAS Rendere PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=REND ``` -- Analyse DUT Renderer Outputs Only (on Reference Platform) +- Analyse BE conformance for DUT Renderer Outputs Only + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=REND --analyse --be-test + ``` + +- Analyse NON-BE conformance DUT Renderer Outputs Only ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=REND --analyse + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=REND --analyse ``` - Run DUT ISAR Encoder Tests Only (on Target Platform) @@ -267,7 +285,13 @@ All CUT tests can be run specifically for IVAS Encoder,IVAS Decoder,IVAS Rendere PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ISAR_ENC ``` -- Analyse DUT ISAR Encoder Outputs Only (on Reference Platform) +- Analyse BE conformance for DUT ISAR Encoder Outputs Only (on Reference Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ISAR_ENC --analyse --be-test + ``` + +- Analyse NON-BE conformance for DUT ISAR Encoder Outputs Only (on Reference Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ISAR_ENC --analyse @@ -279,8 +303,14 @@ All CUT tests can be run specifically for IVAS Encoder,IVAS Decoder,IVAS Rendere PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ISAR ``` -- Analyse DUT ISAR Decoder Outputs Only (on Reference Platform) +- Analyse BE conformance for DUT ISAR Decoder Outputs Only + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=ISAR --analyse --be-test + ``` + +- Analyse NON-BE conformance DUT ISAR Decoder Outputs Only ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ISAR --analyse + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=ISAR --analyse ``` diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 92596403df..bb56bfacc6 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -46,6 +46,7 @@ import scipy.io.wavfile as wav import warnings import math import scipy.signal as sig +import filecmp sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) @@ -246,11 +247,13 @@ class MLDConformance: self.wavdiffbin = os.path.join(self.toolsdir, exe_platform, "wav-diff") self.CutBins = dict() self.mldcsv = dict() + self.BEcsv = dict() self.sampleStats = dict() for tag in IVAS_Bins.keys(): self.CutBins[tag] = os.path.join(self.cut_build_path, IVAS_Bins[tag]) self.mldcsv[tag] = os.path.join(self.outputDir, f"mld_{tag}.csv") + self.BEcsv[tag] = os.path.join(self.outputDir, f"BE_{tag}.csv") self.sampleStats[tag] = os.path.join( self.outputDir, f"sampleStats_{tag}.csv" ) @@ -502,49 +505,63 @@ class MLDConformance: assert isinstance( testDesc, TestDesciptor ), f"Expected pcm test descriptor for {tag}" - ##### skip MLD verification for files with only 1 frame as MLD does not run with such files. Possible solution: append 0s and then compare ##### - if testDesc.rawCmdline.find("_cut.192.fer") == -1: - self.mld( - tag, - dutPytestTag, - refFile=testDesc.refOutput, - dutFile=testDesc.dutOutput, - ) + + if self.args.be_test: + DUTmdFiles = self.getMDfileList(outFile=testDesc.dutOutput) + REFmdFiles = self.getMDfileList(outFile=testDesc.refOutput) + self.beTest(tag, dutPytestTag, refFile=testDesc.refOutput, dutFile=testDesc.dutOutput, DUTmdFileList=DUTmdFiles, REFmdFileList=REFmdFiles ) + else: + ##### skip MLD verification for files with only 1 frame as MLD does not run with such files. Possible solution: append 0s and then compare ##### + if testDesc.rawCmdline.find("_cut.192.fer") == -1: + self.mld( + tag, + dutPytestTag, + refFile=testDesc.refOutput, + dutFile=testDesc.dutOutput, + ) def analyseOneEncoderTest(self, tag: str, encPytestTag: str): testDesc = self.TestDesc[tag][encPytestTag] assert isinstance( testDesc, BitstrmTestDescriptor ), f"Expected bitstream test descriptor for {tag}" - refDecOutputFile = testDesc.refOutput.replace(".192", "_REFDECODED.wav") - dutDecOutputFile = testDesc.dutOutput.replace(".192", "_CUT_REFDECODED.wav") - # Decode the encoded output with Reference IVAS decoder - dutDecCmd = testDesc.refDecCmdline.split()[:-2] + [ - testDesc.dutOutput, - dutDecOutputFile, - ] - dutDecCmd = ["" if x == "-VOIP" else x for x in dutDecCmd] - dutDecCmd = " ".join(dutDecCmd) - self.process(command=dutDecCmd) - self.mld(tag, encPytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile) + if self.args.be_test: + self.beTest(tag, encPytestTag, refFile=testDesc.refOutput, dutFile=testDesc.dutOutput ) + else: + refDecOutputFile = testDesc.refOutput.replace(".192", "_REFDECODED.wav") + + dutDecOutputFile = testDesc.dutOutput.replace(".192", "_CUT_REFDECODED.wav") + # Decode the encoded output with Reference IVAS decoder + dutDecCmd = testDesc.refDecCmdline.split()[:-2] + [ + testDesc.dutOutput, + dutDecOutputFile, + ] + dutDecCmd = ["" if x == "-VOIP" else x for x in dutDecCmd] + dutDecCmd = " ".join(dutDecCmd) + self.process(command=dutDecCmd) + self.mld(tag, encPytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile) def analyseOneIsarEncoderTest(self, tag: str, pytestTag: str): testDesc = self.TestDesc[tag][pytestTag] assert isinstance( testDesc, BitstrmTestDescriptor ), f"Expected bitstream test descriptor for {tag}" - refDecOutputFile = testDesc.refOutput.replace(".splt.bit", ".wav") - dutDecOutputFile = testDesc.dutOutput.replace(".splt.bit", ".wav") - # Decode the encoded output with Reference ISAR decoder - dutDecCmd = testDesc.refDecCmdline.split() - for idx, cmd in enumerate(dutDecCmd): - if cmd == "-o" and (idx + 1) < len(dutDecCmd): - dutDecCmd[idx + 1] = dutDecOutputFile - if cmd == "-i" and (idx + 1) < len(dutDecCmd): - dutDecCmd[idx + 1] = testDesc.dutOutput - self.process(command=" ".join(dutDecCmd)) - self.mld(tag, pytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile) + + if self.args.be_test: + self.beTest(tag, pytestTag, refFile=testDesc.refOutput, dutFile=testDesc.dutOutput ) + else: + refDecOutputFile = testDesc.refOutput.replace(".splt.bit", ".wav") + dutDecOutputFile = testDesc.dutOutput.replace(".splt.bit", ".wav") + # Decode the encoded output with Reference ISAR decoder + dutDecCmd = testDesc.refDecCmdline.split() + for idx, cmd in enumerate(dutDecCmd): + if cmd == "-o" and (idx + 1) < len(dutDecCmd): + dutDecCmd[idx + 1] = dutDecOutputFile + if cmd == "-i" and (idx + 1) < len(dutDecCmd): + dutDecCmd[idx + 1] = testDesc.dutOutput + self.process(command=" ".join(dutDecCmd)) + self.mld(tag, pytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile) def getRendOutputFile(self, command: str): cmds = command.split() @@ -556,6 +573,14 @@ class MLDConformance: def getOutputFile(self, command: str): return command.split()[-1] + def getMDfileList(self, outFile: str): + MDfiles = [] + for i in range(0,3): + MDfiles.append(outFile + '.' + str(i) + '.csv') + MDfiles.append(outFile + '.met') + return MDfiles + + def setCommandExec(self, tag: str, command, ref: bool = False): exec = self.RefBins[tag] if ref else self.CutBins[tag] commands = command.split() @@ -663,9 +688,13 @@ class MLDConformance: def analyseTag(self, tag: str): # reset MLD, Sample Stats - open(self.mldcsv[tag], "w").close() - with open(self.sampleStats[tag], "w") as f: - f.write(f"PYTESTTAG, MAXDIFF, RMSdB, BEFRAMES_PERCENT, MAX_MLD\n") + if self.args.be_test: + with open(self.BEcsv[tag], "w") as f: + f.write(f"PYTESTTAG, BE=0 NON-BE=1\n") + else: + open(self.mldcsv[tag], "w").close() + with open(self.sampleStats[tag], "w") as f: + f.write(f"PYTESTTAG, MAXDIFF, RMSdB, BEFRAMES_PERCENT, MAX_MLD\n") selectedTests = list() if self.filter: for pyTestsTag in self.TestDesc[tag].keys(): @@ -685,7 +714,10 @@ class MLDConformance: else: for pyTestsTag in selectedTests: self.analyseOneCommand(tag, pyTestsTag) - self.doAnalysis(selectTag=tag) + if self.args.be_test: + self.doBEanalysis(selectTag=tag) + else: + self.doAnalysis(selectTag=tag) def process(self, command) -> int: if self.args.verbose: @@ -795,6 +827,42 @@ class MLDConformance: f"{pytestTag}, {maxDiff}, {rmsdB}, {beSamplesPercent}, {mldThisFile.max()}\n" ) + def beTest(self, tag, pytestTag, refFile, dutFile, DUTmdFileList=[], REFmdFileList=[]): + BE_flag = 0 + if not filecmp.cmp(refFile, dutFile): + BE_flag = 1 + with open(self.BEcsv[tag], "a") as f: + f.write( + f"{pytestTag}, {BE_flag}\n" + ) + + for i in range(0,len(DUTmdFileList)): + if os.path.exists(DUTmdFileList[i]): + BE_flag = 0 + if not filecmp.cmp(REFmdFileList[i], DUTmdFileList[i]): + BE_flag = 1 + with open(self.BEcsv[tag], "a") as f: + f.write( + f"{DUTmdFileList[i]}, {BE_flag}\n" + ) + + + def doBEanalysis(self, selectTag="all"): + keys = IVAS_Bins.keys() if selectTag == "all" else [selectTag] + for tag in keys: + if os.path.exists(self.BEcsv[tag]): + BEresult = np.loadtxt( + self.BEcsv[tag], + delimiter=",", + dtype=int, + skiprows=1, + usecols=1, + ) + if np.sum(BEresult) > 0: + print(f"<{tag}> FAILED BE TEST, check {self.BEcsv[tag]}") + else: + print(f"<{tag}> PASSED BE TEST") + def doAnalysis(self, selectTag="all"): keys = IVAS_Bins.keys() if selectTag == "all" else [selectTag] for tag in keys: @@ -901,6 +969,12 @@ if __name__ == "__main__": default="ALL", help='Choose tests to run ["ENC", "DEC", "REND", "ISAR", "ISAR_ENC", "ALL"]', ) + parser.add_argument( + "--be-test", + default=False, + action="store_true", + help='runs only BE tests', + ) parser.add_argument( "--no-multi-processing", default=False, -- GitLab