From 30254a5db536e360329db4b833d048e090cd7eec Mon Sep 17 00:00:00 2001 From: Ripinder Singh Date: Fri, 17 Oct 2025 22:17:56 +1100 Subject: [PATCH 1/9] Initial implementation of the conformance script for frame based MLD Signed-off-by: Ripinder Singh --- scripts/ivas_conformance/runConformance.py | 558 +++++++++++++++++++++ 1 file changed, 558 insertions(+) create mode 100644 scripts/ivas_conformance/runConformance.py diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py new file mode 100644 index 0000000000..9df23416c8 --- /dev/null +++ b/scripts/ivas_conformance/runConformance.py @@ -0,0 +1,558 @@ +#!/usr/bin/env python3 + +""" +(C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" +import argparse +import os +import platform +import re +import numpy as np +import subprocess +import tempfile +import sys +from typing import Optional +from multiprocessing import Process, Value + +sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) + +from pyaudio3dtools.audiofile import readfile, writefile +from pyaudio3dtools.audioarray import resample + + +class MLDConformance: + IVAS_Bins = { + "ENC": "IVAS_cod", + "DEC": "IVAS_dec", + "REND": "IVAS_rend", + "ISAR": "ISAR_post_rend", + } + + def __init__(self, args) -> None: + self.RefBins = dict() + self.CutBins = dict() + self.Commands = dict() + self.multiprocessing = not args.no_multi_processing + self.dryrun = args.dryrun + self.verbose = args.verbose + self.executedTests = Value("i", 0) + self.failedTests = Value("i", 0) + self.testvecDir = args.testvecDir + self.ref_build_path = args.ref_build_path + self.cut_build_path = args.cut_build_path + self.filter = args.filter + self.EncoderToDecoderCmdMap = dict() + + self.scriptsDir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + self.outputDir = os.path.join(self.scriptsDir, "CUT_OUTPUTS") + self.toolsdir = os.path.join(self.scriptsDir, "tools") + self.testvDir = os.path.join(self.testvecDir, "testv") + self.refDir = self.testvDir + self.mldbin = os.path.join(self.toolsdir, platform.system(), "mld") + + self.logFile = os.path.join(self.outputDir, "runlog.txt") + self.failedCmdsFile = os.path.join(self.outputDir, "failedCmds.txt") + open(self.logFile, "w").close() + open(self.failedCmdsFile, "w").close() + + self.mldcsv = dict() + self.sampleStats = dict() + for tag in MLDConformance.IVAS_Bins.keys(): + self.mldcsv[tag] = os.path.join(self.outputDir, f"mld_{tag}.csv") + self.sampleStats[tag] = os.path.join(self.outputDir, f"sampleStats_{tag}.csv") + + self.setup() + + def createDirs(self): + os.makedirs(self.outputDir, exist_ok=True) + subdirs = ["enc", "dec", "renderer", "split_rendering"] + for odir in subdirs: + os.makedirs(os.path.join(self.outputDir, "ref", odir), exist_ok=True) + os.makedirs(os.path.join(self.outputDir, "dut", odir), exist_ok=True) + + def setup(self): + self.createDirs() + for tag in MLDConformance.IVAS_Bins.keys(): + self.RefBins[tag] = os.path.join( + self.ref_build_path, MLDConformance.IVAS_Bins[tag] + ) + self.CutBins[tag] = os.path.join( + self.cut_build_path, MLDConformance.IVAS_Bins[tag] + ) + self.Commands[tag] = list() + + def accumulateCommands(self): + for root, _, files in os.walk(self.testvecDir): + for file_name in files: + basename, ext = os.path.splitext(file_name) + if ( + ("Readme_IVAS_" in basename) + and ext == ".txt" + and not ("ISAR" in basename) + ): + print(f"Accumulating commands from {file_name}") + file = os.path.join(root, file_name) + self.parseCommandsFile(file) + self.mapEncoderToDecoderCommands() + print("No of tests :") + for key in self.Commands.keys(): + print(f" {key} : {len(self.Commands[key])}") + + def parseCommandsFile(self, filePath): + with open(filePath) as fp: + for line in fp.readlines(): + m = re.search(r"^\$(CUT_.+_BIN) ", line) + if m: + tag = m.group(1).split("_")[1] + if tag in self.Commands.keys(): + self.Commands[tag].append(line) + + def getPcmPytestTag(self, command: str) -> str: + decInput = ( + os.path.basename(command.split()[-2]) + .replace(".fer", "") + .replace("_cut", "") + ) + return decInput.split(".")[-2] + + def getEncPytestTag(self, command: str) -> str: + return os.path.basename(command.split()[-1]).split(".")[-2] + + def mapEncoderToDecoderCommands(self): + decoderPyTestTags = dict() + encoderPyTestTags = dict() + for idx, command in enumerate(self.Commands["DEC"]): + decoderPyTestTags[self.getPcmPytestTag(command)] = idx + for idx, command in enumerate(self.Commands["ENC"]): + encoderPyTestTags[self.getEncPytestTag(command)] = idx + + for encTag in encoderPyTestTags.keys(): + if encTag in decoderPyTestTags.keys(): + self.EncoderToDecoderCmdMap[encoderPyTestTags[encTag]] = ( + decoderPyTestTags[encTag] + ) + if self.verbose: + print( + f"{encTag} {encoderPyTestTags[encTag]} -> {decoderPyTestTags[encTag]}" + ) + print(f"{self.Commands['ENC'][encoderPyTestTags[encTag]]}") + print(f"{self.Commands['DEC'][decoderPyTestTags[encTag]]}") + else: + print(f"{encTag} not fount in decoder") + print( + f"Mapped decoder tests for {len(self.EncoderToDecoderCmdMap)} encoder tests out of {len(self.Commands['ENC'])} tests" + ) + assert len(self.EncoderToDecoderCmdMap) == len( + self.Commands["ENC"] + ), "Failed to Map Encoder Commands to Decoder Commands" + + def runOneEncoderTest(self, command: str): + encPytestTag = self.getEncPytestTag(command) + refCommand = self.reformatCommand(command=command, ref=True) + refEncOutput = self.getOutputFile(refCommand) + refCommand = self.setCommandExec(tag="ENC", command=refCommand, ref=True) + self.process(command=refCommand) + + # Run reference Encoder + encCommandIdx = self.Commands["ENC"].index(command) + command = self.reformatCommand(command=command, ref=False) + command = self.setCommandExec(tag="ENC", command=command, ref=False) + dutEncOutput = self.getOutputFile(command) + self.process(command=command) + assert ".192" in dutEncOutput, "Output file not identified" + + # Decode the encoded output with Reference decoder + dutDecOutputFile = dutEncOutput.replace(".192", "_CUT_REFDECODED.wav") + refDecOutputFile = dutEncOutput.replace(".192", "_REF_REFDECODED.wav") + decCommandIdx = self.EncoderToDecoderCmdMap[encCommandIdx] + command = self.reformatCommand( + command=self.Commands["DEC"][decCommandIdx], ref=False + ) + command = command.replace("-VOIP", "") + dutDecCmd = ( + [self.RefBins["DEC"]] + + command.split()[1:-2] + + [dutEncOutput, dutDecOutputFile] + ) + refDecCmd = ( + [self.RefBins["DEC"]] + + command.split()[1:-2] + + [refEncOutput, refDecOutputFile] + ) + self.process(command=" ".join(dutDecCmd)) + self.process(command=" ".join(refDecCmd)) + self.mld("ENC", encPytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile) + + def runOneDecoderTest(self, tag: str, command: str): + dutPytestTag = self.getPcmPytestTag(command) + refInputFile = command.split()[-2].replace( + "$REF_PATH/ref", f"{self.testvDir}/ref" + ) + refInputFile = refInputFile.replace("_cut.192.fer", ".192") + # refInputFile = refInputFile.replace(".fer.192", ".192").replace(".192.fer", ".192").replace("_cut.192.fer", ".192").replace("_cut.192", ".192") + refDecOutput = self.getOutputFile(command).replace( + "$CUT_PATH/dut", f"{self.testvDir}/ref" + ) + command = self.reformatCommand(command=command, ref=False) + # command = command.replace("-VOIP", "") + dutDecOutputFile = self.getOutputFile(command) + dutDecCmd = ( + [self.CutBins["DEC"]] + + command.split()[1:-2] + + [refInputFile, dutDecOutputFile] + ) + self.process(command=" ".join(dutDecCmd)) + self.mld("DEC", dutPytestTag, refFile=refDecOutput, dutFile=dutDecOutputFile) + + def getRendOutputFile(self, command: str): + cmds = command.split() + for idx, cmd in enumerate(cmds): + if cmd == "-o" and (idx + 1) < len(cmds): + return cmds[idx + 1] + assert False, "Outputname not found" + + def runOneRendererTest(self, tag: str, command: str): + refRendOutputFile = self.getRendOutputFile(command).replace( + "$CUT_PATH/renderer", f"{self.testvDir}/renderer" + ) + rendPytestTag = os.path.basename(refRendOutputFile).split(".")[-2] + command = self.reformatCommand(command=command, ref=False) + dutRendCmd = " ".join([self.CutBins["REND"]] + command.split()[1:]) + dutRendOutputFile = self.getRendOutputFile(dutRendCmd) + self.process(command=dutRendCmd) + self.mld("REND", rendPytestTag, refFile=refRendOutputFile, dutFile=dutRendOutputFile) + + def getOutputFile(self, command: str): + return command.split()[-1] + + def setCommandExec(self, tag: str, command, ref: bool): + exec = self.RefBins[tag] if ref else self.CutBins[tag] + commands = command.split() + return " ".join([exec, *commands[1:]]) + + def reformatCommand(self, command: str, ref: bool) -> str: + command = command.replace("$TESTV_PATH", self.testvecDir) + command = command.replace( + "$REF_PATH/split_rendering", f"{self.testvecDir}/testv/split_rendering" + ) + + ################ HACKS ######################### + command = command.replace("_cut.192.fer", ".192") + command = command.replace("_cut.192", ".192") + command = command.replace(".fer.192", ".192") + command = command.replace(".192.fer", ".192") + ################################################## + + command = command.replace( + "$REF_PATH/ref/param_file/enc/", f"{self.outputDir}/ref/enc/" + ) + command = command.replace( + "$REF_PATH/ref/param_file/dec/", f"{self.outputDir}/ref/dec/" + ) + command = command.replace( + "$REF_PATH/ref/sba_bs/pkt/", f"{self.outputDir}/ref/enc/" + ) + + if ref: + command = command.replace( + "$CUT_PATH/dut/param_file/enc/", f"{self.outputDir}/ref/enc/" + ) + command = command.replace( + "$CUT_PATH/dut/param_file/dec/", f"{self.outputDir}/ref/dec/" + ) + command = command.replace( + "$CUT_PATH/renderer/cut/", f"{self.outputDir}/ref/renderer/" + ) + command = command.replace( + "$CUT_PATH/split_rendering/cut/", + f"{self.outputDir}/ref/split_rendering/", + ) + command = command.replace( + "$CUT_PATH/dut/sba_bs/pkt/", f"{self.outputDir}/ref/enc/" + ) + command = command.replace( + "$CUT_PATH/dut/sba_bs/raw/", f"{self.outputDir}/ref/dec/" + ) + else: + command = command.replace( + "$CUT_PATH/dut/param_file/enc/", f"{self.outputDir}/dut/enc/" + ) + command = command.replace( + "$CUT_PATH/dut/param_file/dec/", f"{self.outputDir}/dut/dec/" + ) + command = command.replace( + "$CUT_PATH/renderer/cut/", f"{self.outputDir}/dut/renderer/" + ) + command = command.replace( + "$CUT_PATH/split_rendering/cut/", + f"{self.outputDir}/dut/split_rendering/", + ) + command = command.replace( + "$CUT_PATH/dut/sba_bs/pkt/", f"{self.outputDir}/dut/enc/" + ) + command = command.replace( + "$CUT_PATH/dut/sba_bs/raw/", f"{self.outputDir}/dut/dec/" + ) + + return command + + def runOneCommand(self, tag: str, command: str, ref: bool): + if tag == "ENC": + self.runOneEncoderTest(command) + elif tag == "DEC": + self.runOneDecoderTest(tag, command) + elif tag == "REND": + self.runOneRendererTest(tag, command) + else: + assert False, f"Un-implemented Tag {tag}" + self.executedTests.value += 1 + self.stats() + + def runTag(self, tag: str, ref: bool = False): + self.executedTests.value = 0 + self.failedTests.value = 0 + # reset MLD, Sample Stats + open(self.mldcsv[tag], "w").close() + with open(self.sampleStats[tag], "w") as f: + f.write(f"PYTESTTAG, MAXDIFF, RMSdB, BEFRAMES_PERCENT, MAX_MLD\n") + + processes = list() # Multiprocess list + commands = list() + if self.filter: + for command in self.Commands[tag]: + if self.filter in command: + commands.append(command) + else: + commands = self.Commands[tag] + + self.totalTests = len(commands) + print( + f"Executing tests for {tag} {'Filter='+self.filter if self.filter else ''} ({self.totalTests} tests)" + ) + if self.multiprocessing: + for command in commands: + p = Process( + target=self.runOneCommand, + args=(tag, command, ref), + ) + processes.append(p) + p.start() + for p in processes: + p.join() + else: + for command in commands: + self.runOneCommand(tag, command, ref) + + def process(self, command) -> int: + if self.verbose: + print(command) + with open(self.logFile, "a") as fd: + fd.write(command + "\n") + + with open(self.logFile, "a") as fd: + if not self.dryrun: + c = subprocess.run( + command, stdout=fd, stderr=subprocess.STDOUT, text=True, shell=True + ) + if c.returncode: + with open(self.failedCmdsFile, "a") as f: + f.write(command + "\n") + self.failedTests.value += 1 + # c.check_returncode() + return 0 + + def stats(self): + print( + f"Executed: {self.executedTests.value} / {self.totalTests} Failed: {self.failedTests.value}", + end="\r", + ) + + def getSampleStats(self, refSamples: np.ndarray, dutSamples: np.ndarray): + nSamples = min(refSamples.shape[0], dutSamples.shape[0]) + diff = (refSamples[:nSamples] / 32768.0) - (dutSamples[:nSamples] / 32768.0) + beSamplesPercent = ( + 100.0 * np.sum(diff == 0) / refSamples.shape[0] / refSamples.shape[1] + ) + maxDiff = np.abs(diff).max() + rmsdB = int(10.0 * np.log10(np.average(diff**2) + np.finfo(np.float64).eps) * 10) / 10.0 + return (maxDiff, rmsdB, beSamplesPercent) + + + def mld(self, tag, pytestTag, refFile, dutFile): + mldThisFile = np.zeros(0) + with tempfile.TemporaryDirectory() as tmpdir: + refSamples, fsR = readfile(refFile, outdtype="float") + dutSamples, fsD = readfile(dutFile, outdtype="float") + assert ( + refSamples.shape[1] == dutSamples.shape[1] + ), "No of channels mismatch if ref vs cut" + maxDiff, rmsdB, beSamplesPercent = self.getSampleStats(refSamples, dutSamples) + + nChans = refSamples.shape[1] + + if fsR != 48000: + refSamples = np.clip(resample(refSamples, fsR, 48000), -32768, 32767) + if fsD != 48000: + dutSamples = np.clip(resample(dutSamples, fsD, 48000), -32768, 32767) + + for ch in range(nChans): + mldFile = os.path.join( + tmpdir, f"{tempfile.gettempprefix()}_ch{ch}_MLD.csv" + ) + refFileMono = os.path.join( + tmpdir, os.path.basename(refFile).replace(".wav", f"_ch{ch}.wav") + ) + dutFileMono = os.path.join( + tmpdir, os.path.basename(dutFile).replace(".wav", f"_ch{ch}.wav") + ) + writefile(refFileMono, refSamples[:, ch], 48000) + writefile(dutFileMono, dutSamples[:, ch], 48000) + command = [ + self.mldbin, + "-f", + "20", + "-o", + mldFile, + "-s", + refFileMono, + dutFileMono, + ] + self.process(" ".join(command)) + mldThisChan = np.loadtxt(mldFile, delimiter=" ", dtype=float) + if ch == 0: + mldThisFile = mldThisChan + else: + mldThisFile = np.maximum(mldThisFile, mldThisChan) + + if mldThisFile.size > 0: + with open(self.mldcsv[tag], "ab") as f: + np.savetxt(f, mldThisFile, delimiter=",") + with open(self.sampleStats[tag], "a") as f: + f.write(f"{pytestTag}, {maxDiff}, {rmsdB}, {beSamplesPercent}, {mldThisFile.max()}\n") + + + def doAnalysis(self, selectTag = "all"): + keys = MLDConformance.IVAS_Bins.keys() if selectTag == "all" else [ selectTag ] + for tag in keys: + if os.path.exists(self.mldcsv[tag]): + mdlValues = np.loadtxt(self.mldcsv[tag], delimiter=" ", dtype=float) + N = mdlValues.shape[0] + if N == 0: + continue + m0 = np.sum(mdlValues == 0) + m1 = np.sum(mdlValues <= 1.0) + m2 = np.sum(mdlValues <= 2.0) + m5 = np.sum(mdlValues <= 5.0) + + PCNT = lambda num: int(1000 * num / N) / 10.0 + print(f"\n##########################################################") + print(f"<{tag}> Total Frames: {N}") + print(f"<{tag}> MAX MLD across all frames : {mdlValues.max()}") + print(f"<{tag}> Frames with MLD == 0 : {m0} frames ({PCNT(m0)}%)") + print(f"<{tag}> Frames with MLD <= 1 : {m1} frames ({PCNT(m1)}%)") + print(f"<{tag}> Frames with MLD <= 2 : {m2} frames ({PCNT(m2)}%)") + print(f"<{tag}> Frames with MLD <= 5 : {m5} frames ({PCNT(m5)}%)") + print("##########################################################\n") + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Compare .wav files in two folders using mld per frame" + ) + + parser.add_argument( + "--testvecDir", + type=str, + required=True, + help="Base folder containing a set of conformance scripts named Readme_IVAS_xxxx.txt", + ) + parser.add_argument( + "--ref_build_path", + type=str, + required=True, + help="Path to the reference build folder containing IVAS Encoder, Decoder, Renderer and Post Render binaries", + ) + parser.add_argument( + "--cut_build_path", + type=str, + required=True, + help="Path to the CUT build folder containing IVAS Encoder, Decoder, Renderer and Post Render binaries", + ) + parser.add_argument( + "--verbose", + default=False, + action="store_true", + help="Enable verbose printing", + ) + parser.add_argument( + "--dryrun", + default=False, + action="store_true", + help="Do not run any processing, just dump command lines to runlog", + ) + + parser.add_argument( + "--filter", + type=str, + default=None, + help="Filter test based on text provided", + ) + parser.add_argument( + "--test-mode", + type=str, + default="ALL", + help='Choose tests to run ["ENC", "DEC", "REND", "ISAR", "ALL"]', + ) + parser.add_argument( + "--no-multi-processing", + default=False, + action="store_true", + help="Disable multi-processing for sequential test run (debugging)", + ) + parser.add_argument( + "--analyse-only", + default=False, + action="store_true", + help="Do not run DUT, use existing mld and bitdiff stats files to generate analysis only", + ) + + args = parser.parse_args() + + conformance = MLDConformance(args) + + conformance.accumulateCommands() + # import sys + # sys.exit(0) + testTags = MLDConformance.IVAS_Bins.keys() if args.test_mode == "ALL" else [args.test_mode] + for tag in testTags: + if tag == "ISAR": + # Not implemented yet + continue + if not args.analyse_only: + conformance.runTag(tag, ref=True) + conformance.doAnalysis(selectTag=tag) -- GitLab From 139ee0c774669b706af654556133fa7a5c8b6396 Mon Sep 17 00:00:00 2001 From: Ripinder Singh Date: Mon, 27 Oct 2025 16:19:17 +1100 Subject: [PATCH 2/9] Add option to regenerate encoder references * Format with black Signed-off-by: Ripinder Singh --- scripts/ivas_conformance/runConformance.py | 61 ++++++++++++++++------ 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 9df23416c8..7e03f3239d 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -59,6 +59,7 @@ class MLDConformance: self.CutBins = dict() self.Commands = dict() self.multiprocessing = not args.no_multi_processing + self.regenEncRefs = args.regenerate_enc_refs self.dryrun = args.dryrun self.verbose = args.verbose self.executedTests = Value("i", 0) @@ -85,7 +86,9 @@ class MLDConformance: self.sampleStats = dict() for tag in MLDConformance.IVAS_Bins.keys(): self.mldcsv[tag] = os.path.join(self.outputDir, f"mld_{tag}.csv") - self.sampleStats[tag] = os.path.join(self.outputDir, f"sampleStats_{tag}.csv") + self.sampleStats[tag] = os.path.join( + self.outputDir, f"sampleStats_{tag}.csv" + ) self.setup() @@ -173,11 +176,21 @@ class MLDConformance: ), "Failed to Map Encoder Commands to Decoder Commands" def runOneEncoderTest(self, command: str): + encPytestTag = self.getEncPytestTag(command) - refCommand = self.reformatCommand(command=command, ref=True) - refEncOutput = self.getOutputFile(refCommand) - refCommand = self.setCommandExec(tag="ENC", command=refCommand, ref=True) - self.process(command=refCommand) + + if self.regenEncRefs: + refCommand = self.reformatCommand(command=command, ref=True) + refEncOutput = self.getOutputFile(refCommand) + self.process( + command=self.setCommandExec(tag="ENC", command=refCommand, ref=True) + ) + else: + refEncOutput = self.getOutputFile(command) + refEncOutput = refEncOutput.replace( + "$CUT_PATH/dut/param_file/enc/", + f"{self.testvecDir}/testv/ref/param_file/enc/", + ) # Run reference Encoder encCommandIdx = self.Commands["ENC"].index(command) @@ -207,7 +220,9 @@ class MLDConformance: ) self.process(command=" ".join(dutDecCmd)) self.process(command=" ".join(refDecCmd)) - self.mld("ENC", encPytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile) + self.mld( + "ENC", encPytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile + ) def runOneDecoderTest(self, tag: str, command: str): dutPytestTag = self.getPcmPytestTag(command) @@ -246,7 +261,9 @@ class MLDConformance: dutRendCmd = " ".join([self.CutBins["REND"]] + command.split()[1:]) dutRendOutputFile = self.getRendOutputFile(dutRendCmd) self.process(command=dutRendCmd) - self.mld("REND", rendPytestTag, refFile=refRendOutputFile, dutFile=dutRendOutputFile) + self.mld( + "REND", rendPytestTag, refFile=refRendOutputFile, dutFile=dutRendOutputFile + ) def getOutputFile(self, command: str): return command.split()[-1] @@ -400,10 +417,12 @@ class MLDConformance: 100.0 * np.sum(diff == 0) / refSamples.shape[0] / refSamples.shape[1] ) maxDiff = np.abs(diff).max() - rmsdB = int(10.0 * np.log10(np.average(diff**2) + np.finfo(np.float64).eps) * 10) / 10.0 + rmsdB = ( + int(10.0 * np.log10(np.average(diff**2) + np.finfo(np.float64).eps) * 10) + / 10.0 + ) return (maxDiff, rmsdB, beSamplesPercent) - def mld(self, tag, pytestTag, refFile, dutFile): mldThisFile = np.zeros(0) with tempfile.TemporaryDirectory() as tmpdir: @@ -412,7 +431,9 @@ class MLDConformance: assert ( refSamples.shape[1] == dutSamples.shape[1] ), "No of channels mismatch if ref vs cut" - maxDiff, rmsdB, beSamplesPercent = self.getSampleStats(refSamples, dutSamples) + maxDiff, rmsdB, beSamplesPercent = self.getSampleStats( + refSamples, dutSamples + ) nChans = refSamples.shape[1] @@ -454,11 +475,12 @@ class MLDConformance: with open(self.mldcsv[tag], "ab") as f: np.savetxt(f, mldThisFile, delimiter=",") with open(self.sampleStats[tag], "a") as f: - f.write(f"{pytestTag}, {maxDiff}, {rmsdB}, {beSamplesPercent}, {mldThisFile.max()}\n") - + f.write( + f"{pytestTag}, {maxDiff}, {rmsdB}, {beSamplesPercent}, {mldThisFile.max()}\n" + ) - def doAnalysis(self, selectTag = "all"): - keys = MLDConformance.IVAS_Bins.keys() if selectTag == "all" else [ selectTag ] + def doAnalysis(self, selectTag="all"): + keys = MLDConformance.IVAS_Bins.keys() if selectTag == "all" else [selectTag] for tag in keys: if os.path.exists(self.mldcsv[tag]): mdlValues = np.loadtxt(self.mldcsv[tag], delimiter=" ", dtype=float) @@ -480,6 +502,7 @@ class MLDConformance: print(f"<{tag}> Frames with MLD <= 5 : {m5} frames ({PCNT(m5)}%)") print("##########################################################\n") + if __name__ == "__main__": parser = argparse.ArgumentParser( description="Compare .wav files in two folders using mld per frame" @@ -503,6 +526,12 @@ if __name__ == "__main__": required=True, help="Path to the CUT build folder containing IVAS Encoder, Decoder, Renderer and Post Render binaries", ) + parser.add_argument( + "--regenerate_enc_refs", + default=False, + action="store_true", + help="Enable verbose printing", + ) parser.add_argument( "--verbose", default=False, @@ -548,7 +577,9 @@ if __name__ == "__main__": conformance.accumulateCommands() # import sys # sys.exit(0) - testTags = MLDConformance.IVAS_Bins.keys() if args.test_mode == "ALL" else [args.test_mode] + testTags = ( + MLDConformance.IVAS_Bins.keys() if args.test_mode == "ALL" else [args.test_mode] + ) for tag in testTags: if tag == "ISAR": # Not implemented yet -- GitLab From dc04bedd7edded390fe16318c4def3c51fa90301 Mon Sep 17 00:00:00 2001 From: rtyag Date: Tue, 28 Oct 2025 17:23:16 +1100 Subject: [PATCH 3/9] minor fix to delete cut output and recreate with every run, remove -f option with mld --- scripts/ivas_conformance/runConformance.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 7e03f3239d..18b8f02ff8 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -39,6 +39,7 @@ import tempfile import sys from typing import Optional from multiprocessing import Process, Value +import shutil sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) @@ -72,6 +73,9 @@ class MLDConformance: self.scriptsDir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) self.outputDir = os.path.join(self.scriptsDir, "CUT_OUTPUTS") + if os.path.exists(self.outputDir): + shutil.rmtree(self.outputDir, ignore_errors=False) + os.mkdir(self.outputDir) self.toolsdir = os.path.join(self.scriptsDir, "tools") self.testvDir = os.path.join(self.testvecDir, "testv") self.refDir = self.testvDir @@ -456,8 +460,6 @@ class MLDConformance: writefile(dutFileMono, dutSamples[:, ch], 48000) command = [ self.mldbin, - "-f", - "20", "-o", mldFile, "-s", -- GitLab From 545adc3013a5938eb3a6ef3200cc1c9939a68576 Mon Sep 17 00:00:00 2001 From: rtyag Date: Tue, 28 Oct 2025 21:35:28 +1100 Subject: [PATCH 4/9] fix no multi processing arg --- scripts/ivas_conformance/runConformance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 18b8f02ff8..6c29cd0094 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -560,7 +560,7 @@ if __name__ == "__main__": help='Choose tests to run ["ENC", "DEC", "REND", "ISAR", "ALL"]', ) parser.add_argument( - "--no-multi-processing", + "--no_multi_processing", default=False, action="store_true", help="Disable multi-processing for sequential test run (debugging)", -- GitLab From 5d9d0ec21f2e7e6b5c5ae924a644dccd4112044d Mon Sep 17 00:00:00 2001 From: rtyag Date: Fri, 31 Oct 2025 20:47:41 +1100 Subject: [PATCH 5/9] update the script to work with latest BE conformance changes --- scripts/ivas_conformance/runConformance.py | 31 +++++++++++++--------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 6c29cd0094..7841b4c7b9 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -192,9 +192,13 @@ class MLDConformance: else: refEncOutput = self.getOutputFile(command) refEncOutput = refEncOutput.replace( - "$CUT_PATH/dut/param_file/enc/", + "$CUT_PATH/ref/param_file/enc/", f"{self.testvecDir}/testv/ref/param_file/enc/", ) + refEncOutput = refEncOutput.replace( + "$CUT_PATH/ref/sba_bs/pkt/", + f"{self.testvecDir}/testv/ref/sba_bs/pkt/", + ) # Run reference Encoder encCommandIdx = self.Commands["ENC"].index(command) @@ -233,10 +237,10 @@ class MLDConformance: refInputFile = command.split()[-2].replace( "$REF_PATH/ref", f"{self.testvDir}/ref" ) - refInputFile = refInputFile.replace("_cut.192.fer", ".192") + #refInputFile = refInputFile.replace("_cut.192.fer", ".192") # refInputFile = refInputFile.replace(".fer.192", ".192").replace(".192.fer", ".192").replace("_cut.192.fer", ".192").replace("_cut.192", ".192") refDecOutput = self.getOutputFile(command).replace( - "$CUT_PATH/dut", f"{self.testvDir}/ref" + "$CUT_PATH/ref", f"{self.testvDir}/ref" ) command = self.reformatCommand(command=command, ref=False) # command = command.replace("-VOIP", "") @@ -247,7 +251,10 @@ class MLDConformance: + [refInputFile, dutDecOutputFile] ) self.process(command=" ".join(dutDecCmd)) - self.mld("DEC", dutPytestTag, refFile=refDecOutput, dutFile=dutDecOutputFile) + + ##### skip MLD verification for files with only 1 frame as MLD does not run with such files. Possible solution: append 0s and then compare ##### + if refInputFile.find("_cut.192.fer") == -1: + self.mld("DEC", dutPytestTag, refFile=refDecOutput, dutFile=dutDecOutputFile) def getRendOutputFile(self, command: str): cmds = command.split() @@ -302,10 +309,10 @@ class MLDConformance: if ref: command = command.replace( - "$CUT_PATH/dut/param_file/enc/", f"{self.outputDir}/ref/enc/" + "$CUT_PATH/ref/param_file/enc/", f"{self.outputDir}/ref/enc/" ) command = command.replace( - "$CUT_PATH/dut/param_file/dec/", f"{self.outputDir}/ref/dec/" + "$CUT_PATH/ref/param_file/dec/", f"{self.outputDir}/ref/dec/" ) command = command.replace( "$CUT_PATH/renderer/cut/", f"{self.outputDir}/ref/renderer/" @@ -315,17 +322,17 @@ class MLDConformance: f"{self.outputDir}/ref/split_rendering/", ) command = command.replace( - "$CUT_PATH/dut/sba_bs/pkt/", f"{self.outputDir}/ref/enc/" + "$CUT_PATH/ref/sba_bs/pkt/", f"{self.outputDir}/ref/enc/" ) command = command.replace( - "$CUT_PATH/dut/sba_bs/raw/", f"{self.outputDir}/ref/dec/" + "$CUT_PATH/ref/sba_bs/raw/", f"{self.outputDir}/ref/dec/" ) else: command = command.replace( - "$CUT_PATH/dut/param_file/enc/", f"{self.outputDir}/dut/enc/" + "$CUT_PATH/ref/param_file/enc/", f"{self.outputDir}/dut/enc/" ) command = command.replace( - "$CUT_PATH/dut/param_file/dec/", f"{self.outputDir}/dut/dec/" + "$CUT_PATH/ref/param_file/dec/", f"{self.outputDir}/dut/dec/" ) command = command.replace( "$CUT_PATH/renderer/cut/", f"{self.outputDir}/dut/renderer/" @@ -335,10 +342,10 @@ class MLDConformance: f"{self.outputDir}/dut/split_rendering/", ) command = command.replace( - "$CUT_PATH/dut/sba_bs/pkt/", f"{self.outputDir}/dut/enc/" + "$CUT_PATH/ref/sba_bs/pkt/", f"{self.outputDir}/dut/enc/" ) command = command.replace( - "$CUT_PATH/dut/sba_bs/raw/", f"{self.outputDir}/dut/dec/" + "$CUT_PATH/ref/sba_bs/raw/", f"{self.outputDir}/dut/dec/" ) return command -- GitLab From b4e4cb983a9ebc2d5906095faeda564d5487b079 Mon Sep 17 00:00:00 2001 From: rtyag Date: Fri, 31 Oct 2025 21:00:10 +1100 Subject: [PATCH 6/9] adding a shell script to generate BE conformance package --- ivas_be_conf_test_gen.sh | 43 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 ivas_be_conf_test_gen.sh diff --git a/ivas_be_conf_test_gen.sh b/ivas_be_conf_test_gen.sh new file mode 100644 index 0000000000..f5d4c524f6 --- /dev/null +++ b/ivas_be_conf_test_gen.sh @@ -0,0 +1,43 @@ +make -f Makefile clean +make -f Makefile -j CLANG=0 +cp IVAS_cod IVAS_cod_ref +cp IVAS_dec IVAS_dec_ref +cp IVAS_rend IVAS_rend_ref +cp ISAR_post_rend ISAR_post_rend_ref +python3 scripts/prepare_combined_format_inputs.py +python3 -m pytest -q tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html +python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt +rm -rf testvec +mkdir testvec +mkdir testvec/binauralRenderer_interface +mkdir testvec/testv +mkdir testvec/testv/renderer +mkdir testvec/testv/split_rendering +mkdir testvec/bin +cp scripts/testv/* testvec/testv +cp -r scripts/ls_layouts testvec +cp -r scripts/object_edit testvec +cp -r scripts/switchPaths testvec +cp -r scripts/trajectories testvec +cp -r scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface +cp -r tests/ref testvec/testv/ref +cp -r tests/dut/* testvec/testv/ref +cp -r tests/renderer/cut testvec/testv/renderer/ref +cp -r tests/split_rendering/cut testvec/testv/split_rendering/ref +cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs +cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns + +python3 scripts/cleanup_26252.py + +cp -r tests/conformance-test testvec/ +cp Readme_IVAS_dec.txt testvec +cp Readme_IVAS_enc.txt testvec +cp Readme_IVAS_rend.txt testvec +cp Readme_IVAS_JBM_dec.txt testvec +cp Readme_IVAS_ISAR_dec.txt testvec +cp Readme_IVAS_ISAR_post_rend.txt testvec + +cp IVAS_cod testvec/bin +cp IVAS_dec testvec/bin +cp IVAS_rend testvec/bin +cp ISAR_post_rend testvec/bin -- GitLab From 6416a60c0057007b70a78769efae1d0f2a857fc9 Mon Sep 17 00:00:00 2001 From: rtyag Date: Fri, 31 Oct 2025 23:26:53 +1100 Subject: [PATCH 7/9] add CLANG O0 option to makefile, fix renderer tests partially --- Makefile | 3 +++ ivas_be_conf_test_gen.sh | 27 +++++++++++----------- scripts/ivas_conformance/runConformance.py | 8 +++---- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index c72aaacd8d..48c102c6ab 100644 --- a/Makefile +++ b/Makefile @@ -71,6 +71,9 @@ LDLIBS += -lm # Clang sanitizer compiler options CCCLANG = clang +ifeq "$(CLANG)" "0" +CC = $(CCCLANG) +endif ifeq "$(CLANG)" "1" CC = $(CCCLANG) CFLAGS += -fsanitize=memory diff --git a/ivas_be_conf_test_gen.sh b/ivas_be_conf_test_gen.sh index f5d4c524f6..2536de1f89 100644 --- a/ivas_be_conf_test_gen.sh +++ b/ivas_be_conf_test_gen.sh @@ -5,27 +5,26 @@ cp IVAS_dec IVAS_dec_ref cp IVAS_rend IVAS_rend_ref cp ISAR_post_rend ISAR_post_rend_ref python3 scripts/prepare_combined_format_inputs.py -python3 -m pytest -q tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html +python3 -m pytest -q tests/codec_be_on_mr_nonselection tests/renderer_short/test_renderer.py tests/split_rendering/test_split_rendering.py -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt rm -rf testvec mkdir testvec mkdir testvec/binauralRenderer_interface mkdir testvec/testv -mkdir testvec/testv/renderer +mkdir testvec/testv/renderer_short mkdir testvec/testv/split_rendering mkdir testvec/bin -cp scripts/testv/* testvec/testv -cp -r scripts/ls_layouts testvec -cp -r scripts/object_edit testvec -cp -r scripts/switchPaths testvec -cp -r scripts/trajectories testvec -cp -r scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface -cp -r tests/ref testvec/testv/ref -cp -r tests/dut/* testvec/testv/ref -cp -r tests/renderer/cut testvec/testv/renderer/ref -cp -r tests/split_rendering/cut testvec/testv/split_rendering/ref -cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs -cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns +cp -r scripts/testv/* testvec/testv +cp -r scripts/ls_layouts testvec +cp -r scripts/object_edit testvec +cp -r scripts/switchPaths testvec +cp -r scripts/trajectories testvec +cp -r scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface +cp -r tests/ref testvec/testv/ref +cp -r tests/renderer_short/ref testvec/testv/renderer_short/ref +cp -r tests/split_rendering/ref testvec/testv/split_rendering/ref +cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs +cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns python3 scripts/cleanup_26252.py diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 7841b4c7b9..00252b52a8 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -98,7 +98,7 @@ class MLDConformance: def createDirs(self): os.makedirs(self.outputDir, exist_ok=True) - subdirs = ["enc", "dec", "renderer", "split_rendering"] + subdirs = ["enc", "dec", "renderer_short", "split_rendering"] for odir in subdirs: os.makedirs(os.path.join(self.outputDir, "ref", odir), exist_ok=True) os.makedirs(os.path.join(self.outputDir, "dut", odir), exist_ok=True) @@ -265,7 +265,7 @@ class MLDConformance: def runOneRendererTest(self, tag: str, command: str): refRendOutputFile = self.getRendOutputFile(command).replace( - "$CUT_PATH/renderer", f"{self.testvDir}/renderer" + "$CUT_PATH/renderer_short", f"{self.testvDir}/renderer_short" ) rendPytestTag = os.path.basename(refRendOutputFile).split(".")[-2] command = self.reformatCommand(command=command, ref=False) @@ -315,7 +315,7 @@ class MLDConformance: "$CUT_PATH/ref/param_file/dec/", f"{self.outputDir}/ref/dec/" ) command = command.replace( - "$CUT_PATH/renderer/cut/", f"{self.outputDir}/ref/renderer/" + "$CUT_PATH/renderer_short/ref/", f"{self.outputDir}/ref/renderer_short/" ) command = command.replace( "$CUT_PATH/split_rendering/cut/", @@ -335,7 +335,7 @@ class MLDConformance: "$CUT_PATH/ref/param_file/dec/", f"{self.outputDir}/dut/dec/" ) command = command.replace( - "$CUT_PATH/renderer/cut/", f"{self.outputDir}/dut/renderer/" + "$CUT_PATH/renderer_short/ref/", f"{self.outputDir}/dut/renderer_short/" ) command = command.replace( "$CUT_PATH/split_rendering/cut/", -- GitLab From 4725761d9ea49d73d58704b51fe9fedc87721fe9 Mon Sep 17 00:00:00 2001 From: Ripinder Singh Date: Mon, 3 Nov 2025 13:02:36 +1100 Subject: [PATCH 8/9] Add reference generation for encoder Signed-off-by: Ripinder Singh --- scripts/ivas_conformance/runConformance.py | 216 ++++++++++++--------- 1 file changed, 124 insertions(+), 92 deletions(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 00252b52a8..15f9fbcd53 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -55,64 +55,67 @@ class MLDConformance: "ISAR": "ISAR_post_rend", } - def __init__(self, args) -> None: - self.RefBins = dict() - self.CutBins = dict() + def setupCommon(self): self.Commands = dict() - self.multiprocessing = not args.no_multi_processing - self.regenEncRefs = args.regenerate_enc_refs - self.dryrun = args.dryrun - self.verbose = args.verbose - self.executedTests = Value("i", 0) - self.failedTests = Value("i", 0) - self.testvecDir = args.testvecDir - self.ref_build_path = args.ref_build_path - self.cut_build_path = args.cut_build_path - self.filter = args.filter self.EncoderToDecoderCmdMap = dict() + for tag in MLDConformance.IVAS_Bins.keys(): + self.Commands[tag] = list() - self.scriptsDir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + # CREATE OUTPUT DIRECTORY STRUCTURE : CLEAN PREV OUTPUT self.outputDir = os.path.join(self.scriptsDir, "CUT_OUTPUTS") if os.path.exists(self.outputDir): shutil.rmtree(self.outputDir, ignore_errors=False) - os.mkdir(self.outputDir) - self.toolsdir = os.path.join(self.scriptsDir, "tools") - self.testvDir = os.path.join(self.testvecDir, "testv") - self.refDir = self.testvDir - self.mldbin = os.path.join(self.toolsdir, platform.system(), "mld") + os.makedirs(self.outputDir, exist_ok=True) + subdirs = ["enc", "dec", "renderer_short", "split_rendering"] + for odir in subdirs: + os.makedirs(os.path.join(self.outputDir, "ref", odir), exist_ok=True) + os.makedirs(os.path.join(self.outputDir, "dut", odir), exist_ok=True) self.logFile = os.path.join(self.outputDir, "runlog.txt") self.failedCmdsFile = os.path.join(self.outputDir, "failedCmds.txt") open(self.logFile, "w").close() open(self.failedCmdsFile, "w").close() + def setupDUT(self): + self.cut_build_path = args.cut_build_path + self.filter = args.filter + self.mldbin = os.path.join(self.toolsdir, platform.system(), "mld") + self.CutBins = dict() self.mldcsv = dict() self.sampleStats = dict() + for tag in MLDConformance.IVAS_Bins.keys(): + self.CutBins[tag] = os.path.join( + self.cut_build_path, MLDConformance.IVAS_Bins[tag] + ) self.mldcsv[tag] = os.path.join(self.outputDir, f"mld_{tag}.csv") self.sampleStats[tag] = os.path.join( self.outputDir, f"sampleStats_{tag}.csv" ) - self.setup() - - def createDirs(self): - os.makedirs(self.outputDir, exist_ok=True) - subdirs = ["enc", "dec", "renderer_short", "split_rendering"] - for odir in subdirs: - os.makedirs(os.path.join(self.outputDir, "ref", odir), exist_ok=True) - os.makedirs(os.path.join(self.outputDir, "dut", odir), exist_ok=True) - - def setup(self): - self.createDirs() + def setupRef(self): + self.RefBins = dict() + self.ref_build_path = self.args.ref_build_path for tag in MLDConformance.IVAS_Bins.keys(): self.RefBins[tag] = os.path.join( self.ref_build_path, MLDConformance.IVAS_Bins[tag] ) - self.CutBins[tag] = os.path.join( - self.cut_build_path, MLDConformance.IVAS_Bins[tag] - ) - self.Commands[tag] = list() + + def setup(self): + self.setupCommon() + self.setupRef() + if not self.args.regenerate_enc_refs: + self.setupDUT() + + def __init__(self, args) -> None: + self.args = args + self.scriptsDir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + self.testvecDir = args.testvecDir + self.toolsdir = os.path.join(self.scriptsDir, "tools") + self.testvDir = os.path.join(self.testvecDir, "testv") + self.executedTests = Value("i", 0) + self.failedTests = Value("i", 0) + self.setup() def accumulateCommands(self): for root, _, files in os.walk(self.testvecDir): @@ -164,7 +167,7 @@ class MLDConformance: self.EncoderToDecoderCmdMap[encoderPyTestTags[encTag]] = ( decoderPyTestTags[encTag] ) - if self.verbose: + if self.args.verbose: print( f"{encTag} {encoderPyTestTags[encTag]} -> {decoderPyTestTags[encTag]}" ) @@ -179,28 +182,63 @@ class MLDConformance: self.Commands["ENC"] ), "Failed to Map Encoder Commands to Decoder Commands" - def runOneEncoderTest(self, command: str): - - encPytestTag = self.getEncPytestTag(command) - - if self.regenEncRefs: - refCommand = self.reformatCommand(command=command, ref=True) - refEncOutput = self.getOutputFile(refCommand) + def genEncoderReferences(self, command: str, encCommandIdx: int): + # RUN ENCODER COMMAND LINE WITH REFERENCE ENCODER + refCommand = self.reformatCommand(command=command, ref=True) + refEncOutput = self.getOutputFile(refCommand) + if not os.path.exists(refEncOutput): self.process( command=self.setCommandExec(tag="ENC", command=refCommand, ref=True) ) + + # FIND CORRESPONDING DECODER COMMAND + decCommandIdx = self.EncoderToDecoderCmdMap[encCommandIdx] + refDecOutputFile = refEncOutput.replace(".192", "_REFDECODED.wav") + + command = self.reformatCommand( + command=self.Commands["DEC"][decCommandIdx], ref=True + ) + command = command.replace("-VOIP", "") + refDecCmd = ( + [self.RefBins["DEC"]] + + command.split()[1:-2] + + [refEncOutput, refDecOutputFile] + ) + self.process(command=" ".join(refDecCmd)) + self.executedTests.value += 1 + self.stats() + + def runReferenceGeneration(self): + processes = list() # Multiprocess list + commands = conformance.Commands["ENC"] + self.totalTests = len(commands) + if not self.args.no_multi_processing: + for commandIdx, command in enumerate(commands): + p = Process( + target=self.genEncoderReferences, args=(command, commandIdx) + ) + processes.append(p) + p.start() + for p in processes: + p.join() else: - refEncOutput = self.getOutputFile(command) - refEncOutput = refEncOutput.replace( - "$CUT_PATH/ref/param_file/enc/", - f"{self.testvecDir}/testv/ref/param_file/enc/", - ) - refEncOutput = refEncOutput.replace( - "$CUT_PATH/ref/sba_bs/pkt/", - f"{self.testvecDir}/testv/ref/sba_bs/pkt/", - ) + for commandIdx, command in enumerate(commands): + conformance.genEncoderReferences(command, commandIdx) + + def runOneEncoderTest(self, command: str): + encPytestTag = self.getEncPytestTag(command) + refEncOutput = self.getOutputFile(command) + refEncOutput = refEncOutput.replace( + "$CUT_PATH/ref/param_file/enc/", + f"{self.testvecDir}/testv/ref/param_file/enc/", + ) + refEncOutput = refEncOutput.replace( + "$CUT_PATH/ref/sba_bs/pkt/", + f"{self.testvecDir}/testv/ref/sba_bs/pkt/", + ) + refDecOutputFile = refEncOutput.replace(".192", "_REFDECODED.wav") - # Run reference Encoder + # Run CUT Encoder encCommandIdx = self.Commands["ENC"].index(command) command = self.reformatCommand(command=command, ref=False) command = self.setCommandExec(tag="ENC", command=command, ref=False) @@ -210,7 +248,6 @@ class MLDConformance: # Decode the encoded output with Reference decoder dutDecOutputFile = dutEncOutput.replace(".192", "_CUT_REFDECODED.wav") - refDecOutputFile = dutEncOutput.replace(".192", "_REF_REFDECODED.wav") decCommandIdx = self.EncoderToDecoderCmdMap[encCommandIdx] command = self.reformatCommand( command=self.Commands["DEC"][decCommandIdx], ref=False @@ -221,13 +258,7 @@ class MLDConformance: + command.split()[1:-2] + [dutEncOutput, dutDecOutputFile] ) - refDecCmd = ( - [self.RefBins["DEC"]] - + command.split()[1:-2] - + [refEncOutput, refDecOutputFile] - ) self.process(command=" ".join(dutDecCmd)) - self.process(command=" ".join(refDecCmd)) self.mld( "ENC", encPytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile ) @@ -237,7 +268,7 @@ class MLDConformance: refInputFile = command.split()[-2].replace( "$REF_PATH/ref", f"{self.testvDir}/ref" ) - #refInputFile = refInputFile.replace("_cut.192.fer", ".192") + # refInputFile = refInputFile.replace("_cut.192.fer", ".192") # refInputFile = refInputFile.replace(".fer.192", ".192").replace(".192.fer", ".192").replace("_cut.192.fer", ".192").replace("_cut.192", ".192") refDecOutput = self.getOutputFile(command).replace( "$CUT_PATH/ref", f"{self.testvDir}/ref" @@ -251,10 +282,12 @@ class MLDConformance: + [refInputFile, dutDecOutputFile] ) self.process(command=" ".join(dutDecCmd)) - + ##### skip MLD verification for files with only 1 frame as MLD does not run with such files. Possible solution: append 0s and then compare ##### if refInputFile.find("_cut.192.fer") == -1: - self.mld("DEC", dutPytestTag, refFile=refDecOutput, dutFile=dutDecOutputFile) + self.mld( + "DEC", dutPytestTag, refFile=refDecOutput, dutFile=dutDecOutputFile + ) def getRendOutputFile(self, command: str): cmds = command.split() @@ -285,7 +318,7 @@ class MLDConformance: return " ".join([exec, *commands[1:]]) def reformatCommand(self, command: str, ref: bool) -> str: - command = command.replace("$TESTV_PATH", self.testvecDir) + command = command.replace("$TESTV_PATH", self.scriptsDir) command = command.replace( "$REF_PATH/split_rendering", f"{self.testvecDir}/testv/split_rendering" ) @@ -296,38 +329,36 @@ class MLDConformance: command = command.replace(".fer.192", ".192") command = command.replace(".192.fer", ".192") ################################################## - command = command.replace( - "$REF_PATH/ref/param_file/enc/", f"{self.outputDir}/ref/enc/" + "$REF_PATH/ref/param_file/", f"{self.testvDir}/ref/param_file/" ) command = command.replace( - "$REF_PATH/ref/param_file/dec/", f"{self.outputDir}/ref/dec/" + "$REF_PATH/ref/sba_bs/pkt/", f"{self.testvDir}/ref/sba_bs/pkt/" ) - command = command.replace( - "$REF_PATH/ref/sba_bs/pkt/", f"{self.outputDir}/ref/enc/" - ) - if ref: command = command.replace( - "$CUT_PATH/ref/param_file/enc/", f"{self.outputDir}/ref/enc/" + "$CUT_PATH/dut/sba_bs/pkt/", f"{self.testvDir}/ref/sba_bs/pkt/" ) command = command.replace( - "$CUT_PATH/ref/param_file/dec/", f"{self.outputDir}/ref/dec/" + "$CUT_PATH/dut/param_file/", f"{self.testvDir}/ref/param_file/" ) command = command.replace( - "$CUT_PATH/renderer_short/ref/", f"{self.outputDir}/ref/renderer_short/" + "$CUT_PATH/ref/param_file/", f"{self.testvDir}/ref/param_file/" ) command = command.replace( - "$CUT_PATH/split_rendering/cut/", - f"{self.outputDir}/ref/split_rendering/", + "$CUT_PATH/renderer_short/ref/", f"{self.testvDir}/ref/renderer_short/" ) command = command.replace( - "$CUT_PATH/ref/sba_bs/pkt/", f"{self.outputDir}/ref/enc/" + "$CUT_PATH/split_rendering/cut/", + f"{self.testvDir}/ref/split_rendering/", ) command = command.replace( - "$CUT_PATH/ref/sba_bs/raw/", f"{self.outputDir}/ref/dec/" + "$CUT_PATH/ref/sba_bs/", f"{self.testvDir}/ref/sba_bs/" ) else: + #command = command.replace( + # "$CUT_PATH/dut/sba_bs/pkt/", f"{self.outputDir}/dut/enc/" + #) command = command.replace( "$CUT_PATH/ref/param_file/enc/", f"{self.outputDir}/dut/enc/" ) @@ -350,7 +381,7 @@ class MLDConformance: return command - def runOneCommand(self, tag: str, command: str, ref: bool): + def runOneCommand(self, tag: str, command: str): if tag == "ENC": self.runOneEncoderTest(command) elif tag == "DEC": @@ -362,7 +393,7 @@ class MLDConformance: self.executedTests.value += 1 self.stats() - def runTag(self, tag: str, ref: bool = False): + def runTag(self, tag: str): self.executedTests.value = 0 self.failedTests.value = 0 # reset MLD, Sample Stats @@ -383,11 +414,11 @@ class MLDConformance: print( f"Executing tests for {tag} {'Filter='+self.filter if self.filter else ''} ({self.totalTests} tests)" ) - if self.multiprocessing: + if not self.args.no_multi_processing: for command in commands: p = Process( target=self.runOneCommand, - args=(tag, command, ref), + args=(tag, command), ) processes.append(p) p.start() @@ -395,16 +426,16 @@ class MLDConformance: p.join() else: for command in commands: - self.runOneCommand(tag, command, ref) + self.runOneCommand(tag, command) def process(self, command) -> int: - if self.verbose: + if self.args.verbose: print(command) with open(self.logFile, "a") as fd: fd.write(command + "\n") with open(self.logFile, "a") as fd: - if not self.dryrun: + if not self.args.dryrun: c = subprocess.run( command, stdout=fd, stderr=subprocess.STDOUT, text=True, shell=True ) @@ -526,20 +557,18 @@ if __name__ == "__main__": parser.add_argument( "--ref_build_path", type=str, - required=True, help="Path to the reference build folder containing IVAS Encoder, Decoder, Renderer and Post Render binaries", ) parser.add_argument( "--cut_build_path", type=str, - required=True, help="Path to the CUT build folder containing IVAS Encoder, Decoder, Renderer and Post Render binaries", ) parser.add_argument( - "--regenerate_enc_refs", + "--regenerate-enc-refs", default=False, action="store_true", - help="Enable verbose printing", + help="Regenerate the encoder reference bitstreams and decoded outputs", ) parser.add_argument( "--verbose", @@ -567,7 +596,7 @@ if __name__ == "__main__": help='Choose tests to run ["ENC", "DEC", "REND", "ISAR", "ALL"]', ) parser.add_argument( - "--no_multi_processing", + "--no-multi-processing", default=False, action="store_true", help="Disable multi-processing for sequential test run (debugging)", @@ -584,8 +613,11 @@ if __name__ == "__main__": conformance = MLDConformance(args) conformance.accumulateCommands() - # import sys - # sys.exit(0) + + if args.regenerate_enc_refs: + conformance.runReferenceGeneration() + sys.exit(0) + testTags = ( MLDConformance.IVAS_Bins.keys() if args.test_mode == "ALL" else [args.test_mode] ) @@ -594,5 +626,5 @@ if __name__ == "__main__": # Not implemented yet continue if not args.analyse_only: - conformance.runTag(tag, ref=True) + conformance.runTag(tag) conformance.doAnalysis(selectTag=tag) -- GitLab From 81139896a192ae92ddcd878a4d1d65dd64ee95a3 Mon Sep 17 00:00:00 2001 From: rtyag Date: Tue, 4 Nov 2025 16:20:39 +1100 Subject: [PATCH 9/9] update the usage steps in readme.md --- scripts/ivas_conformance/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/scripts/ivas_conformance/README.md b/scripts/ivas_conformance/README.md index 43952b6598..e5fb0824ff 100644 --- a/scripts/ivas_conformance/README.md +++ b/scripts/ivas_conformance/README.md @@ -1,3 +1,22 @@ # IVAS conformance scripts This folder contains scripts for running IVAS conformance tests. This is a placeholder file for instructions. + +no-BE conformance USAGE + +Following CMDs needs to be executed from ivas-codec root folder: + +################generate testvec package and encoder refs (temporary step to obtain testvec package, this step will be removed in final delivery) ############ +sh ivas_be_conf_test_gen.sh +PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/ref_bin --cut_build_path=testvec/cut_bin --test-mode=ENC --regenerate-enc-refs + + + +Encoder conformance: + - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/ref_bin --cut_build_path=testvec/cut_bin --test-mode=ENC +Decoder conformance: + - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/ref_bin --cut_build_path=testvec/cut_bin --test-mode=DEC +Renderer conformance: + - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/ref_bin --cut_build_path=testvec/cut_bin --test-mode=REND +Split renderer confomance: + - -- GitLab