From 1c4e9d8fef0c72651dff738878cfad82984371d2 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 10 Dec 2025 11:17:32 +0100 Subject: [PATCH 01/17] add script for collecting USAN errors from logs --- scripts/parse_usan_errors_from_xml_report.py | 106 +++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 scripts/parse_usan_errors_from_xml_report.py diff --git a/scripts/parse_usan_errors_from_xml_report.py b/scripts/parse_usan_errors_from_xml_report.py new file mode 100644 index 0000000000..bdf32fbb44 --- /dev/null +++ b/scripts/parse_usan_errors_from_xml_report.py @@ -0,0 +1,106 @@ +import pandas as pd +from xml.etree import ElementTree +import argparse +from enum import Enum +from typing import List +import re + + +class UsanError: + def __init__(self, err: str, commandlines: dict): + self.err = err + self.commandlines = commandlines + err_lines = err.split("\n") + self.err_location = err_lines[0].strip().split(": runtime error:")[0] + + def __hash__(self): + return hash(self.err_location) + + def __eq__(self, other): + return self.err_location == other.err_location + + def __repr__(self): + return f"" + + def to_dict(self) -> dict: + return { + "location": self.err_location, + "traceback": "\n".join(self.err.split("\n")[1:]), + **self.commandlines, + } + + +def parse_commandlines_from_sysout(sysout: str) -> dict: + commandlines = { + "IVAS_cod": "", + "IVAS_dec": "", + "IVAS_rend": "", + "ISAR_post_rend": "", + } + for line in sysout.splitlines(): + for exe in commandlines: + # search for name of executable in line + # it is repeated in the sanitizer traceback, hence the "not in" part + if re.search(exe, line) is not None and "_start" not in line: + assert commandlines[exe] == "" + commandlines[exe] = line.strip() + break + + return commandlines + + +def parse_errors_from_sysout(sysout: str) -> List[UsanError]: + commandlines = parse_commandlines_from_sysout(sysout) + errors = [] + + class ParserState(Enum): + OUT = 0 + IN = 1 + + pattern = re.compile(r"(lib_.+|apps)\/(.*\.[ch]):(\d+):(\d+): runtime error:") + + state = ParserState.OUT + accu = [] + for line in sysout.splitlines(): + m = re.match(pattern, line.strip()) + if m is not None: + assert state == ParserState.OUT + state = ParserState.IN + accu = [] + + if state == ParserState.IN: + accu.append(line.strip()) + + if line.strip().startswith("SUMMARY:"): + assert state == ParserState.IN + errors.append(UsanError("\n".join(accu), commandlines)) + state = ParserState.OUT + + return errors + + +def main(args): + tree = ElementTree.parse(args.xml_report) + root = tree.getroot() + + errors = [] + for tc in root[0].findall("testcase"): + sysout = tc.find("system-out") + assert sysout is not None + + sysout = sysout.text + errors.extend(parse_errors_from_sysout(sysout)) + + unique_errors = list(set(errors)) + + df = pd.DataFrame([e.to_dict() for e in unique_errors]) + df.to_csv(args.outfile, index=False) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("xml_report") + parser.add_argument("outfile") + + args = parser.parse_args() + main(args) -- GitLab From 01592e5a954f4958a5f09cca1f13d3075c9b526b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 10 Dec 2025 12:57:15 +0100 Subject: [PATCH 02/17] change CI ref --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 81e3d0a3d6..6e0682b0a5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ variables: # note: GitLab cannot reference variables defined by users in the include ref:, we need to use a YAML anchor for this # see https://docs.gitlab.com/ci/yaml/includes/#use-variables-with-include for more information - IVAS_CODEC_CI_REF: &IVAS_CODEC_CI_REF main + IVAS_CODEC_CI_REF: &IVAS_CODEC_CI_REF kiene/ubsan-error-reporting # If you need to set some config variable only in a local branch, then add an overwrite here # One example is DISABLE_HRTF - this will be set on a branch which is about to be merged and will be removed in a subsequent second MR # this is more easily done directly here in the child repo -- GitLab From c1d0be92ddf4cc2120bdfb705928a9d20c0fc8aa Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 10 Dec 2025 14:06:32 +0100 Subject: [PATCH 03/17] handle no-error case (when there is no system-out element) --- scripts/parse_usan_errors_from_xml_report.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scripts/parse_usan_errors_from_xml_report.py b/scripts/parse_usan_errors_from_xml_report.py index bdf32fbb44..7d069fabb8 100644 --- a/scripts/parse_usan_errors_from_xml_report.py +++ b/scripts/parse_usan_errors_from_xml_report.py @@ -85,11 +85,8 @@ def main(args): errors = [] for tc in root[0].findall("testcase"): - sysout = tc.find("system-out") - assert sysout is not None - - sysout = sysout.text - errors.extend(parse_errors_from_sysout(sysout)) + for sysout in tc.findall("system-out"): + errors.extend(parse_errors_from_sysout(sysout.text)) unique_errors = list(set(errors)) -- GitLab From e5511cf2ac04195826e7050a08ba8fe50d4fd080 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 10 Dec 2025 15:05:25 +0100 Subject: [PATCH 04/17] add msan (untested yet) + sorting --- scripts/parse_usan_errors_from_xml_report.py | 73 +++++++++++++++----- 1 file changed, 56 insertions(+), 17 deletions(-) diff --git a/scripts/parse_usan_errors_from_xml_report.py b/scripts/parse_usan_errors_from_xml_report.py index 7d069fabb8..138b3a5688 100644 --- a/scripts/parse_usan_errors_from_xml_report.py +++ b/scripts/parse_usan_errors_from_xml_report.py @@ -6,30 +6,57 @@ from typing import List import re -class UsanError: - def __init__(self, err: str, commandlines: dict): - self.err = err +class SanitizerError: + def __init__(self, traceback: str, commandlines: dict) -> None: + self.traceback = traceback self.commandlines = commandlines - err_lines = err.split("\n") - self.err_location = err_lines[0].strip().split(": runtime error:")[0] + self.location = "" + self.type = "" def __hash__(self): - return hash(self.err_location) + return hash(self.location) def __eq__(self, other): - return self.err_location == other.err_location + return self.location == other.location def __repr__(self): - return f"" + return f"<{self.__class__} at {self.location}>" + + def __lt__(self, other): + return self.location < other.location def to_dict(self) -> dict: return { - "location": self.err_location, - "traceback": "\n".join(self.err.split("\n")[1:]), + "location": self.location, + "type": self.type, + "traceback": self.traceback, **self.commandlines, } +class UsanError(SanitizerError): + def __init__(self, traceback: str, commandlines: dict): + super().__init__(traceback, commandlines) + err_lines = traceback.split("\n") + self.location = err_lines[0].strip().split(": runtime error:")[0] + self.type = "undefined-behaviour" + + +class MsanError(SanitizerError): + def __init__(self, traceback: str, commandlines: dict) -> None: + super().__init__(traceback, commandlines) + err_lines = traceback.split("\n") + last_line = err_lines[-1].strip() + assert last_line.startswith("SUMMARY: MemorySanitizer: ") + m = re.search( + r"^SUMMARY: MemorySanitizer: ([a-z-]*) .*\/(.*\.[ch]:\d+:\d+) in .+$", + traceback, + ) + assert m is not None + + self.type, self.location = m.groups() + + def parse_commandlines_from_sysout(sysout: str) -> dict: commandlines = { "IVAS_cod": "", @@ -57,23 +84,34 @@ def parse_errors_from_sysout(sysout: str) -> List[UsanError]: OUT = 0 IN = 1 - pattern = re.compile(r"(lib_.+|apps)\/(.*\.[ch]):(\d+):(\d+): runtime error:") + pattern_usan = re.compile(r"(lib_.+|apps)\/(.*\.[ch]):(\d+):(\d+): runtime error:") + pattern_msan = re.compile(r" MemorySanitizer: ") state = ParserState.OUT accu = [] - for line in sysout.splitlines(): - m = re.match(pattern, line.strip()) - if m is not None: + err_cls = None + for l in sysout.splitlines(): + line = l.strip() + + m_usan = re.match(pattern_usan, line) + m_msan = re.match(pattern_msan, line) + + assert m_msan != m_usan or (m_usan is None and m_msan is None) + match_found = m_usan is not None or m_msan is not None + + if match_found: assert state == ParserState.OUT state = ParserState.IN accu = [] + err_cls = UsanError if m_usan is not None else MsanError if state == ParserState.IN: - accu.append(line.strip()) + accu.append(line) if line.strip().startswith("SUMMARY:"): assert state == ParserState.IN - errors.append(UsanError("\n".join(accu), commandlines)) + + errors.append(err_cls("\n".join(accu), commandlines)) state = ParserState.OUT return errors @@ -88,7 +126,8 @@ def main(args): for sysout in tc.findall("system-out"): errors.extend(parse_errors_from_sysout(sysout.text)) - unique_errors = list(set(errors)) + unique_errors = list(sorted(set(errors))) + print(f"Found {len(unique_errors)} unique errors") df = pd.DataFrame([e.to_dict() for e in unique_errors]) df.to_csv(args.outfile, index=False) -- GitLab From d6da884c83980ba06b71421c45c0e92ef3d3231c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 10 Dec 2025 15:33:48 +0100 Subject: [PATCH 05/17] unify classes more --- scripts/parse_usan_errors_from_xml_report.py | 36 +++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/scripts/parse_usan_errors_from_xml_report.py b/scripts/parse_usan_errors_from_xml_report.py index 138b3a5688..e696da3182 100644 --- a/scripts/parse_usan_errors_from_xml_report.py +++ b/scripts/parse_usan_errors_from_xml_report.py @@ -2,16 +2,17 @@ import pandas as pd from xml.etree import ElementTree import argparse from enum import Enum -from typing import List +from typing import List, Tuple import re class SanitizerError: + SUMMARY_ID = "" + def __init__(self, traceback: str, commandlines: dict) -> None: self.traceback = traceback self.commandlines = commandlines - self.location = "" - self.type = "" + self.type, self.location = self.parse_type_and_location(traceback) def __hash__(self): return hash(self.location) @@ -33,28 +34,31 @@ class SanitizerError: **self.commandlines, } + def parse_type_and_location(self, traceback) -> Tuple[str, str]: + last_line = traceback.split("\n")[-1].strip() + assert last_line.startswith(f"SUMMARY: {self.SUMMARY_ID}") + m = re.match( + r"SUMMARY: " + self.SUMMARY_ID + r": ([a-z-]*) (.*\/.*\.[ch]:\d+:\d+) in", + last_line, + ) + assert m is not None + + type, location = m.groups() + return type, location + class UsanError(SanitizerError): + SUMMARY_ID = "UndefinedBehaviorSanitizer" + def __init__(self, traceback: str, commandlines: dict): super().__init__(traceback, commandlines) - err_lines = traceback.split("\n") - self.location = err_lines[0].strip().split(": runtime error:")[0] - self.type = "undefined-behaviour" class MsanError(SanitizerError): + SUMMARY_ID = "MemorySanitizer" + def __init__(self, traceback: str, commandlines: dict) -> None: super().__init__(traceback, commandlines) - err_lines = traceback.split("\n") - last_line = err_lines[-1].strip() - assert last_line.startswith("SUMMARY: MemorySanitizer: ") - m = re.search( - r"^SUMMARY: MemorySanitizer: ([a-z-]*) .*\/(.*\.[ch]:\d+:\d+) in .+$", - traceback, - ) - assert m is not None - - self.type, self.location = m.groups() def parse_commandlines_from_sysout(sysout: str) -> dict: -- GitLab From 55ecc3643a3bf026ec29795b10bfc4a13733e866 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 10 Dec 2025 15:52:58 +0100 Subject: [PATCH 06/17] make it work for MSAN --- scripts/parse_usan_errors_from_xml_report.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/scripts/parse_usan_errors_from_xml_report.py b/scripts/parse_usan_errors_from_xml_report.py index e696da3182..4de475c183 100644 --- a/scripts/parse_usan_errors_from_xml_report.py +++ b/scripts/parse_usan_errors_from_xml_report.py @@ -97,13 +97,16 @@ def parse_errors_from_sysout(sysout: str) -> List[UsanError]: for l in sysout.splitlines(): line = l.strip() - m_usan = re.match(pattern_usan, line) - m_msan = re.match(pattern_msan, line) + m_usan = re.search(pattern_usan, line) + m_msan = re.search(pattern_msan, line) - assert m_msan != m_usan or (m_usan is None and m_msan is None) - match_found = m_usan is not None or m_msan is not None + usan_start_found = m_usan is not None + msan_start_found = m_msan is not None and not line.startswith("SUMMARY:") - if match_found: + assert usan_start_found != msan_start_found or ( + not usan_start_found and not msan_start_found + ) + if usan_start_found or msan_start_found: assert state == ParserState.OUT state = ParserState.IN accu = [] @@ -112,7 +115,7 @@ def parse_errors_from_sysout(sysout: str) -> List[UsanError]: if state == ParserState.IN: accu.append(line) - if line.strip().startswith("SUMMARY:"): + if line.startswith("SUMMARY:"): assert state == ParserState.IN errors.append(err_cls("\n".join(accu), commandlines)) -- GitLab From f852e9303a7ba9aaff13c7225f9797456e010161 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 10 Dec 2025 15:56:13 +0100 Subject: [PATCH 07/17] add parsing of other executables --- scripts/parse_usan_errors_from_xml_report.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/parse_usan_errors_from_xml_report.py b/scripts/parse_usan_errors_from_xml_report.py index 4de475c183..75256cb4eb 100644 --- a/scripts/parse_usan_errors_from_xml_report.py +++ b/scripts/parse_usan_errors_from_xml_report.py @@ -64,6 +64,8 @@ class MsanError(SanitizerError): def parse_commandlines_from_sysout(sysout: str) -> dict: commandlines = { "IVAS_cod": "", + "networkSimulator_g192": "", + "eid-xor": "", "IVAS_dec": "", "IVAS_rend": "", "ISAR_post_rend": "", @@ -72,7 +74,12 @@ def parse_commandlines_from_sysout(sysout: str) -> dict: for exe in commandlines: # search for name of executable in line # it is repeated in the sanitizer traceback, hence the "not in" part - if re.search(exe, line) is not None and "_start" not in line: + # the "not at the start" condition is for eid-xor (there are also lines like this: "eid-xor command:") + if ( + re.search(exe, line) is not None + and "_start" not in line + and not line.strip().startswith(exe) + ): assert commandlines[exe] == "" commandlines[exe] = line.strip() break -- GitLab From 0a54e9017f915a697837af4e075f124c349e45af Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 10 Dec 2025 15:58:47 +0100 Subject: [PATCH 08/17] rename script --- ...om_xml_report.py => parse_sanitizer_errors_from_xml_report.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{parse_usan_errors_from_xml_report.py => parse_sanitizer_errors_from_xml_report.py} (100%) diff --git a/scripts/parse_usan_errors_from_xml_report.py b/scripts/parse_sanitizer_errors_from_xml_report.py similarity index 100% rename from scripts/parse_usan_errors_from_xml_report.py rename to scripts/parse_sanitizer_errors_from_xml_report.py -- GitLab From cd1bf3fffd4f1dab91ad0d8b14e8a9bfbb45868e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 10 Dec 2025 16:33:18 +0100 Subject: [PATCH 09/17] add commandline postprocessing --- .../parse_sanitizer_errors_from_xml_report.py | 44 ++++++++++++++++--- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/scripts/parse_sanitizer_errors_from_xml_report.py b/scripts/parse_sanitizer_errors_from_xml_report.py index 75256cb4eb..cfb343a98f 100644 --- a/scripts/parse_sanitizer_errors_from_xml_report.py +++ b/scripts/parse_sanitizer_errors_from_xml_report.py @@ -1,9 +1,13 @@ +#!/usr/env python3 + import pandas as pd from xml.etree import ElementTree import argparse from enum import Enum from typing import List, Tuple import re +import os +from pathlib import Path class SanitizerError: @@ -61,7 +65,7 @@ class MsanError(SanitizerError): super().__init__(traceback, commandlines) -def parse_commandlines_from_sysout(sysout: str) -> dict: +def parse_commandlines_from_sysout(sysout: str, cwd: Path) -> dict: commandlines = { "IVAS_cod": "", "networkSimulator_g192": "", @@ -81,14 +85,38 @@ def parse_commandlines_from_sysout(sysout: str) -> dict: and not line.strip().startswith(exe) ): assert commandlines[exe] == "" - commandlines[exe] = line.strip() + commandlines[exe] = postprocess_cmdline(line.strip(), cwd) break return commandlines -def parse_errors_from_sysout(sysout: str) -> List[UsanError]: - commandlines = parse_commandlines_from_sysout(sysout) +def postprocess_cmdline(cmdline: str, cwd: Path) -> str: + cmdline_split = cmdline.split() + cmdline_proc = [] + + # change absolute paths into relative ones + # remove the "quite" flag + # for output and bitstream files only keep the filename + for elem in cmdline_split: + if elem == "-q": + continue + elif (elem_as_path := Path(elem)).is_absolute(): + if elem_as_path.suffix == ".192" or ( + elem_as_path.suffix == ".wav" + and cmdline_split.index(elem) == len(cmdline_split) - 1 + ): + cmdline_proc.append(elem_as_path.name) + else: + cmdline_proc.append(str(elem_as_path.relative_to(cwd))) + else: + cmdline_proc.append(elem) + + return " ".join(cmdline_proc) + + +def parse_errors_from_sysout(sysout: str, cwd: Path) -> List[UsanError]: + commandlines = parse_commandlines_from_sysout(sysout, cwd) errors = [] class ParserState(Enum): @@ -138,7 +166,7 @@ def main(args): errors = [] for tc in root[0].findall("testcase"): for sysout in tc.findall("system-out"): - errors.extend(parse_errors_from_sysout(sysout.text)) + errors.extend(parse_errors_from_sysout(sysout.text, args.inject_cwd)) unique_errors = list(sorted(set(errors))) print(f"Found {len(unique_errors)} unique errors") @@ -151,6 +179,12 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("xml_report") parser.add_argument("outfile") + parser.add_argument( + "--inject_cwd", + help="Use this as cwd when pruning the long paths in the command lines. Debug option for testing.", + default=Path(os.getcwd()).absolute(), + type=Path, + ) args = parser.parse_args() main(args) -- GitLab From 6bdf4359ccc7bd69d267b699b19c63fdc3deb1ae Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 10 Dec 2025 16:37:36 +0100 Subject: [PATCH 10/17] add testcase name --- .../parse_sanitizer_errors_from_xml_report.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/scripts/parse_sanitizer_errors_from_xml_report.py b/scripts/parse_sanitizer_errors_from_xml_report.py index cfb343a98f..29ab9a4cb2 100644 --- a/scripts/parse_sanitizer_errors_from_xml_report.py +++ b/scripts/parse_sanitizer_errors_from_xml_report.py @@ -13,9 +13,10 @@ from pathlib import Path class SanitizerError: SUMMARY_ID = "" - def __init__(self, traceback: str, commandlines: dict) -> None: + def __init__(self, traceback: str, commandlines: dict, testcase: str) -> None: self.traceback = traceback self.commandlines = commandlines + self.testcase = testcase self.type, self.location = self.parse_type_and_location(traceback) def __hash__(self): @@ -32,6 +33,7 @@ class SanitizerError: def to_dict(self) -> dict: return { + "testcase": self.testcase, "location": self.location, "type": self.type, "traceback": self.traceback, @@ -54,16 +56,10 @@ class SanitizerError: class UsanError(SanitizerError): SUMMARY_ID = "UndefinedBehaviorSanitizer" - def __init__(self, traceback: str, commandlines: dict): - super().__init__(traceback, commandlines) - class MsanError(SanitizerError): SUMMARY_ID = "MemorySanitizer" - def __init__(self, traceback: str, commandlines: dict) -> None: - super().__init__(traceback, commandlines) - def parse_commandlines_from_sysout(sysout: str, cwd: Path) -> dict: commandlines = { @@ -115,7 +111,9 @@ def postprocess_cmdline(cmdline: str, cwd: Path) -> str: return " ".join(cmdline_proc) -def parse_errors_from_sysout(sysout: str, cwd: Path) -> List[UsanError]: +def parse_errors_from_sysout( + sysout: str, testcase_name: str, cwd: Path +) -> List[UsanError]: commandlines = parse_commandlines_from_sysout(sysout, cwd) errors = [] @@ -153,7 +151,7 @@ def parse_errors_from_sysout(sysout: str, cwd: Path) -> List[UsanError]: if line.startswith("SUMMARY:"): assert state == ParserState.IN - errors.append(err_cls("\n".join(accu), commandlines)) + errors.append(err_cls("\n".join(accu), commandlines, testcase_name)) state = ParserState.OUT return errors @@ -165,8 +163,11 @@ def main(args): errors = [] for tc in root[0].findall("testcase"): + tc_name = tc.attrib["name"] for sysout in tc.findall("system-out"): - errors.extend(parse_errors_from_sysout(sysout.text, args.inject_cwd)) + errors.extend( + parse_errors_from_sysout(sysout.text, tc_name, args.inject_cwd) + ) unique_errors = list(sorted(set(errors))) print(f"Found {len(unique_errors)} unique errors") -- GitLab From 358560c212ab1d01a3ea46666ffbe1c02c5568c6 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 10 Dec 2025 16:50:29 +0100 Subject: [PATCH 11/17] make command lines parsing more precise there is a bw sw pattern with "_start" in it ... --- scripts/parse_sanitizer_errors_from_xml_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/parse_sanitizer_errors_from_xml_report.py b/scripts/parse_sanitizer_errors_from_xml_report.py index 29ab9a4cb2..a610bdf022 100644 --- a/scripts/parse_sanitizer_errors_from_xml_report.py +++ b/scripts/parse_sanitizer_errors_from_xml_report.py @@ -77,7 +77,7 @@ def parse_commandlines_from_sysout(sysout: str, cwd: Path) -> dict: # the "not at the start" condition is for eid-xor (there are also lines like this: "eid-xor command:") if ( re.search(exe, line) is not None - and "_start" not in line + and " in _start " not in line and not line.strip().startswith(exe) ): assert commandlines[exe] == "" -- GitLab From 2936ec6e0c29c638383b8f6b6365039f7b8718ca Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 11 Dec 2025 11:01:31 +0100 Subject: [PATCH 12/17] fix sanitizer type name --- scripts/parse_sanitizer_errors_from_xml_report.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/parse_sanitizer_errors_from_xml_report.py b/scripts/parse_sanitizer_errors_from_xml_report.py index a610bdf022..8537ddb8b4 100644 --- a/scripts/parse_sanitizer_errors_from_xml_report.py +++ b/scripts/parse_sanitizer_errors_from_xml_report.py @@ -26,7 +26,7 @@ class SanitizerError: return self.location == other.location def __repr__(self): - return f"<{self.__class__} at {self.location}>" + return f"<{self.__class__.__name__} at {self.location}>" def __lt__(self, other): return self.location < other.location @@ -34,6 +34,7 @@ class SanitizerError: def to_dict(self) -> dict: return { "testcase": self.testcase, + "sanitizer": self.__class__.__name__.replace("Error", "").upper(), "location": self.location, "type": self.type, "traceback": self.traceback, -- GitLab From bd6698b2ea09e698dd8e43148c86497d1f107365 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 11 Dec 2025 16:33:36 +0100 Subject: [PATCH 13/17] handle absolute paths in location --- scripts/parse_sanitizer_errors_from_xml_report.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/parse_sanitizer_errors_from_xml_report.py b/scripts/parse_sanitizer_errors_from_xml_report.py index 8537ddb8b4..bc80d3b0e4 100644 --- a/scripts/parse_sanitizer_errors_from_xml_report.py +++ b/scripts/parse_sanitizer_errors_from_xml_report.py @@ -13,11 +13,13 @@ from pathlib import Path class SanitizerError: SUMMARY_ID = "" - def __init__(self, traceback: str, commandlines: dict, testcase: str) -> None: + def __init__( + self, traceback: str, commandlines: dict, testcase: str, cwd: Path = Path(".") + ) -> None: self.traceback = traceback self.commandlines = commandlines self.testcase = testcase - self.type, self.location = self.parse_type_and_location(traceback) + self.type, self.location = self.parse_type_and_location(traceback, cwd) def __hash__(self): return hash(self.location) @@ -41,7 +43,7 @@ class SanitizerError: **self.commandlines, } - def parse_type_and_location(self, traceback) -> Tuple[str, str]: + def parse_type_and_location(self, traceback, cwd) -> Tuple[str, str]: last_line = traceback.split("\n")[-1].strip() assert last_line.startswith(f"SUMMARY: {self.SUMMARY_ID}") m = re.match( @@ -51,6 +53,9 @@ class SanitizerError: assert m is not None type, location = m.groups() + + if Path(location).is_absolute(): + location = str(Path(location).relative_to(cwd)) return type, location @@ -152,7 +157,7 @@ def parse_errors_from_sysout( if line.startswith("SUMMARY:"): assert state == ParserState.IN - errors.append(err_cls("\n".join(accu), commandlines, testcase_name)) + errors.append(err_cls("\n".join(accu), commandlines, testcase_name, cwd)) state = ParserState.OUT return errors -- GitLab From 7f18a2a899843808fbfe685b76c32cb5b3fc5e8e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 12 Dec 2025 09:44:22 +0100 Subject: [PATCH 14/17] print errors in failing renderer calls to add to system-out --- tests/renderer/utils.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 1aa7370f08..f7c1b38462 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -67,7 +67,8 @@ from pyaudio3dtools.audiofile import readfile from ..cmp_pcm import cmp_pcm from ..conftest import get_split_idx, parse_properties -def _run_cmd(cmd, test_info=None, env=None ): + +def _run_cmd(cmd, test_info=None, env=None): """ Helper function for running some command. Raises a SystemError if either the return code is non-zero or a USAN printout is detected @@ -78,6 +79,7 @@ def _run_cmd(cmd, test_info=None, env=None ): # check for USAN error first if "UndefinedBehaviorSanitizer" in stdout: error = f"USAN error detected in stdout of command: {' '.join(cmd)}\n{stdout}" + print(error) if test_info is not None: test_info.error = error raise SystemError(error) @@ -87,6 +89,7 @@ def _run_cmd(cmd, test_info=None, env=None ): proc.check_returncode() except sp.CalledProcessError as e: error = f"Command returned non-zero exit status ({e.returncode}): {' '.join(e.cmd)}\n{e.stderr}\n{e.stdout}" + print(error) if test_info is not None: test_info.error = error raise SystemError(error) @@ -106,6 +109,7 @@ def run_ivas_isar_enc_cmd(cmd, test_info=None, env=None): logging.info(f"\nRunning IVAS ISAR encoder command\n{' '.join(cmd)}\n") _run_cmd(cmd, test_info=test_info, env=env) + def run_ivas_isar_dec_cmd(cmd, test_info=None, env=None): if BIN_SUFFIX_MERGETARGET in cmd[0]: logging.info(f"\nREF decoder command:\n\t{' '.join(cmd)}\n") @@ -113,6 +117,7 @@ def run_ivas_isar_dec_cmd(cmd, test_info=None, env=None): logging.info(f"\nDUT decoder command:\n\t{' '.join(cmd)}\n") _run_cmd(cmd, test_info=test_info, env=env) + def run_isar_post_rend_cmd(cmd, test_info=None, env=None): logging.info(f"\nRunning ISAR post renderer command\n{' '.join(cmd)}\n") _run_cmd(cmd, test_info=test_info, env=env) @@ -771,19 +776,19 @@ def scene_description_file(in_fmt, metadata_tmp, n_obj, input_file, in_meta_file metadata_tmp ).parent # File names must be relative to config file location fp_meta.write(f"{os.path.relpath(input_file, currdir)}\n") # Input file - fp_meta.write(f"{n_obj+1}\n") # Number of sources + fp_meta.write(f"{n_obj + 1}\n") # Number of sources for n in range(0, n_obj): if in_meta_files[n] == "NULL": md_file = "1\n1,0,0" # NULL metadata position: azim=0,elev=0 for 1 frame, looped throughout all frames. else: md_file = os.path.relpath(in_meta_files[n], currdir) - fp_meta.write(f"ISM\n{n+1}\n{md_file}\n") # ISM metadata + fp_meta.write(f"ISM\n{n + 1}\n{md_file}\n") # ISM metadata if "OSBA" in in_fmt: fp_meta.write( "gain_dB:-6\n" ) # Set -6 dB on all components for OSBA to match IVAS_dec fp_meta.write(f"{in_fmt.split('_')[0][1:]}\n") # SBA or MASA - fp_meta.write(f"{n_obj+1}\n") + fp_meta.write(f"{n_obj + 1}\n") fp_meta.write(f"{in_fmt.split('_')[-1]}\n") # SBA or MASA parameter if "OMASA" in in_fmt: fp_meta.write( -- GitLab From 534cbff90337e78e2c0ecdf7bb691e987ca8b27b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 12 Dec 2025 09:46:19 +0100 Subject: [PATCH 15/17] handle tmp files for netsimtrace correctly --- scripts/parse_sanitizer_errors_from_xml_report.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/parse_sanitizer_errors_from_xml_report.py b/scripts/parse_sanitizer_errors_from_xml_report.py index bc80d3b0e4..7971837b4c 100644 --- a/scripts/parse_sanitizer_errors_from_xml_report.py +++ b/scripts/parse_sanitizer_errors_from_xml_report.py @@ -104,9 +104,13 @@ def postprocess_cmdline(cmdline: str, cwd: Path) -> str: if elem == "-q": continue elif (elem_as_path := Path(elem)).is_absolute(): - if elem_as_path.suffix == ".192" or ( - elem_as_path.suffix == ".wav" - and cmdline_split.index(elem) == len(cmdline_split) - 1 + if ( + elem_as_path.suffix == ".192" + or elem_as_path.suffix == ".netsimtrace" + or ( + elem_as_path.suffix == ".wav" + and cmdline_split.index(elem) == len(cmdline_split) - 1 + ) ): cmdline_proc.append(elem_as_path.name) else: -- GitLab From 9523179cdbb060f6bc2467d5e13a7d52e3abf142 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 12 Dec 2025 09:50:53 +0100 Subject: [PATCH 16/17] Revert "print errors in failing renderer calls to add to system-out" This reverts commit 7f18a2a899843808fbfe685b76c32cb5b3fc5e8e. --- tests/renderer/utils.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index f7c1b38462..1aa7370f08 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -67,8 +67,7 @@ from pyaudio3dtools.audiofile import readfile from ..cmp_pcm import cmp_pcm from ..conftest import get_split_idx, parse_properties - -def _run_cmd(cmd, test_info=None, env=None): +def _run_cmd(cmd, test_info=None, env=None ): """ Helper function for running some command. Raises a SystemError if either the return code is non-zero or a USAN printout is detected @@ -79,7 +78,6 @@ def _run_cmd(cmd, test_info=None, env=None): # check for USAN error first if "UndefinedBehaviorSanitizer" in stdout: error = f"USAN error detected in stdout of command: {' '.join(cmd)}\n{stdout}" - print(error) if test_info is not None: test_info.error = error raise SystemError(error) @@ -89,7 +87,6 @@ def _run_cmd(cmd, test_info=None, env=None): proc.check_returncode() except sp.CalledProcessError as e: error = f"Command returned non-zero exit status ({e.returncode}): {' '.join(e.cmd)}\n{e.stderr}\n{e.stdout}" - print(error) if test_info is not None: test_info.error = error raise SystemError(error) @@ -109,7 +106,6 @@ def run_ivas_isar_enc_cmd(cmd, test_info=None, env=None): logging.info(f"\nRunning IVAS ISAR encoder command\n{' '.join(cmd)}\n") _run_cmd(cmd, test_info=test_info, env=env) - def run_ivas_isar_dec_cmd(cmd, test_info=None, env=None): if BIN_SUFFIX_MERGETARGET in cmd[0]: logging.info(f"\nREF decoder command:\n\t{' '.join(cmd)}\n") @@ -117,7 +113,6 @@ def run_ivas_isar_dec_cmd(cmd, test_info=None, env=None): logging.info(f"\nDUT decoder command:\n\t{' '.join(cmd)}\n") _run_cmd(cmd, test_info=test_info, env=env) - def run_isar_post_rend_cmd(cmd, test_info=None, env=None): logging.info(f"\nRunning ISAR post renderer command\n{' '.join(cmd)}\n") _run_cmd(cmd, test_info=test_info, env=env) @@ -776,19 +771,19 @@ def scene_description_file(in_fmt, metadata_tmp, n_obj, input_file, in_meta_file metadata_tmp ).parent # File names must be relative to config file location fp_meta.write(f"{os.path.relpath(input_file, currdir)}\n") # Input file - fp_meta.write(f"{n_obj + 1}\n") # Number of sources + fp_meta.write(f"{n_obj+1}\n") # Number of sources for n in range(0, n_obj): if in_meta_files[n] == "NULL": md_file = "1\n1,0,0" # NULL metadata position: azim=0,elev=0 for 1 frame, looped throughout all frames. else: md_file = os.path.relpath(in_meta_files[n], currdir) - fp_meta.write(f"ISM\n{n + 1}\n{md_file}\n") # ISM metadata + fp_meta.write(f"ISM\n{n+1}\n{md_file}\n") # ISM metadata if "OSBA" in in_fmt: fp_meta.write( "gain_dB:-6\n" ) # Set -6 dB on all components for OSBA to match IVAS_dec fp_meta.write(f"{in_fmt.split('_')[0][1:]}\n") # SBA or MASA - fp_meta.write(f"{n_obj + 1}\n") + fp_meta.write(f"{n_obj+1}\n") fp_meta.write(f"{in_fmt.split('_')[-1]}\n") # SBA or MASA parameter if "OMASA" in in_fmt: fp_meta.write( -- GitLab From ef22f78eadb0e0bd51f3d529836a0fdfb3b974ec Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 12 Dec 2025 10:45:39 +0100 Subject: [PATCH 17/17] some fixes - fix commandline extraction for multiple occurences of same exe - when sorting issues put the ones with more found cmdlines first - add logging --- .../parse_sanitizer_errors_from_xml_report.py | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/scripts/parse_sanitizer_errors_from_xml_report.py b/scripts/parse_sanitizer_errors_from_xml_report.py index 7971837b4c..82628c749b 100644 --- a/scripts/parse_sanitizer_errors_from_xml_report.py +++ b/scripts/parse_sanitizer_errors_from_xml_report.py @@ -8,6 +8,10 @@ from typing import List, Tuple import re import os from pathlib import Path +import logging + + +logging.basicConfig(level=logging.INFO) class SanitizerError: @@ -31,7 +35,14 @@ class SanitizerError: return f"<{self.__class__.__name__} at {self.location}>" def __lt__(self, other): - return self.location < other.location + # order by string comparison of location as first criterion + # if location is the same in both instances, the smaller one is the one with more found command lines + if self.location != other.location: + return self.location < other.location + else: + num_cmdl_self = list(self.commandlines.values()).count("") + num_cmdl_other = list(other.commandlines.values()).count("") + return num_cmdl_self > num_cmdl_other def to_dict(self) -> dict: return { @@ -86,8 +97,14 @@ def parse_commandlines_from_sysout(sysout: str, cwd: Path) -> dict: and " in _start " not in line and not line.strip().startswith(exe) ): - assert commandlines[exe] == "" - commandlines[exe] = postprocess_cmdline(line.strip(), cwd) + if commandlines[exe] != "": + logging.debug( + f"Commandline for {exe} already found, skip second one." + ) + else: + commandlines[exe] = postprocess_cmdline(line.strip(), cwd) + + # assumption: only one commandline per line break return commandlines @@ -124,6 +141,7 @@ def postprocess_cmdline(cmdline: str, cwd: Path) -> str: def parse_errors_from_sysout( sysout: str, testcase_name: str, cwd: Path ) -> List[UsanError]: + logging.debug(testcase_name) commandlines = parse_commandlines_from_sysout(sysout, cwd) errors = [] @@ -179,7 +197,7 @@ def main(args): parse_errors_from_sysout(sysout.text, tc_name, args.inject_cwd) ) - unique_errors = list(sorted(set(errors))) + unique_errors = list(sorted(set(sorted(errors)))) print(f"Found {len(unique_errors)} unique errors") df = pd.DataFrame([e.to_dict() for e in unique_errors]) -- GitLab