From 44ca1b3215f3bf0cd8f9f03c92a7c853a0a57386 Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 29 Jan 2024 18:11:40 +0100 Subject: [PATCH 1/4] change file collection --- ci/collect_artifacts.py | 29 +++++------------------------ ci/run_scheduled_sanitizer_test.py | 4 +--- 2 files changed, 6 insertions(+), 27 deletions(-) diff --git a/ci/collect_artifacts.py b/ci/collect_artifacts.py index d3228ba2e8..49cf3565cb 100755 --- a/ci/collect_artifacts.py +++ b/ci/collect_artifacts.py @@ -15,39 +15,20 @@ def main(args): collect_for_sanitizer_test(file) -def find_failed_files_for_sanitizer_test( - console_log: list, subfolder: str, which="LOGS" -) -> dict(): +def find_failed_files_for_sanitizer_test(console_log: list) -> dict(): - assert which in ["LOGS", "FILE_BASENAMES"] - - pattern_line = "(Encoding|Decoding) failed .*for \/.*(CLANG.|VALGRIND)\/(.*)" - pattern_file = "(.*_b[0-9]*_.*_rs|.*_b[0-9]*_.*_cbr).*" + pattern_line = r"(CLANG.) reports . error\(s\) for (.*)" files_found = dict() for line in console_log: m_line = re.match(pattern_line, line) if m_line is not None: - _, test, filename = m_line.groups() - filename = pathlib.Path(filename).name - m_file = re.match(pattern_file, filename) - if m_file is None: - print(f"Unexpected: no match on {filename} with {pattern_file} - skip") - continue - filename_start = m_file.groups()[0] - - if which == "LOGS": - folder = pathlib.Path(f"{test}/{subfolder}/") - files = [ - f for f in folder.iterdir() if f.name.startswith(filename_start) - ] - elif which == "FILE_BASENAMES": - files = [filename_start] + test, filename = m_line.groups() if test in files_found: - files_found[test].extend(files) + files_found[test].append(filename) else: - files_found[test] = files + files_found[test] = [filename] return files_found diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index 33148897ca..84601b1708 100755 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -179,9 +179,7 @@ def run_check(in_format: str, out_formats: list, tests: list, run_fec: bool = Tr # delete bitstream files for all failed modes to prevent follow-up errors in decoder-only run with open(CONSOLE_OUT_FILE) as f: console_log = f.readlines() - failed_files = find_failed_files_for_sanitizer_test( - console_log, "logs", "FILE_BASENAMES" - ) + failed_files = find_failed_files_for_sanitizer_test(console_log) for t in failed_files.keys(): bs_folder = pathlib.Path(f"{t}/enc") file_starts = failed_files[t] -- GitLab From 8a9b8ecfa3646476dd931aece3d465b2455a4cf8 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 22 Feb 2024 10:36:41 +0100 Subject: [PATCH 2/4] split collected logs by PLC/noPLC --- ci/collect_artifacts.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/ci/collect_artifacts.py b/ci/collect_artifacts.py index e665d75f16..29705dfc3d 100755 --- a/ci/collect_artifacts.py +++ b/ci/collect_artifacts.py @@ -37,16 +37,20 @@ def collect_for_sanitizer_test(file): with open(file) as f: console_log = f.readlines() - files_to_archive_noPLC = find_failed_files_for_sanitizer_test( - console_log, "logs_noPLC" - ) - files_to_archive = find_failed_files_for_sanitizer_test(console_log, "logs") + start_indicators = ["Adding config" in l for l in console_log] + idx_first_run = start_indicators.index(True) + idx_second_run = start_indicators[idx_first_run + 1:].index(True) + no_plc_part = console_log[idx_first_run:idx_second_run] + plc_part = console_log[idx_second_run:] + + files_to_archive_noPLC = find_failed_files_for_sanitizer_test(no_plc_part) + files_to_archive_PLC = find_failed_files_for_sanitizer_test(plc_part) log_folder = pathlib.Path("./LOGS_PLC") log_folder.mkdir() - for test in files_to_archive.keys(): + for test in files_to_archive_PLC.keys(): log_folder.joinpath(test).mkdir() - for test, files in files_to_archive.items(): + for test, files in files_to_archive_PLC.items(): folder = log_folder.joinpath(test) for p in files: source = pathlib.Path(p) -- GitLab From 0196b5dfe1a8a627c9b2e4bd5edb7410715392df Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 22 Feb 2024 10:41:11 +0100 Subject: [PATCH 3/4] replace subfolder name for noPLC log file copying --- ci/collect_artifacts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/collect_artifacts.py b/ci/collect_artifacts.py index 29705dfc3d..ee5700b24a 100755 --- a/ci/collect_artifacts.py +++ b/ci/collect_artifacts.py @@ -64,7 +64,7 @@ def collect_for_sanitizer_test(file): for test, files in files_to_archive_noPLC.items(): folder = log_folder_noPLC.joinpath(test) for p in files: - source = pathlib.Path(p) + source = pathlib.Path(p.replace("/logs/", "/logs_noPLC/")) target = folder.joinpath(source.name) source.rename(target) -- GitLab From 3434463896c498597734a137b959090dd3b81e16 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 22 Feb 2024 11:04:23 +0100 Subject: [PATCH 4/4] fix indexing --- ci/collect_artifacts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/collect_artifacts.py b/ci/collect_artifacts.py index ee5700b24a..c67445e51c 100755 --- a/ci/collect_artifacts.py +++ b/ci/collect_artifacts.py @@ -39,7 +39,7 @@ def collect_for_sanitizer_test(file): start_indicators = ["Adding config" in l for l in console_log] idx_first_run = start_indicators.index(True) - idx_second_run = start_indicators[idx_first_run + 1:].index(True) + idx_second_run = start_indicators[idx_first_run + 1:].index(True) + idx_first_run + 1 no_plc_part = console_log[idx_first_run:idx_second_run] plc_part = console_log[idx_second_run:] -- GitLab