From fd1cab575498efe0fc9a63bb499169c78b3cbc86 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 13 Dec 2024 10:53:18 +0100 Subject: [PATCH 01/46] revert changes to jbm BE test coming in from main --- tests/test_be_for_jbm_neutral_dly_profile.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_be_for_jbm_neutral_dly_profile.py b/tests/test_be_for_jbm_neutral_dly_profile.py index 61bb168a2f..743a4e0cfb 100644 --- a/tests/test_be_for_jbm_neutral_dly_profile.py +++ b/tests/test_be_for_jbm_neutral_dly_profile.py @@ -17,7 +17,7 @@ DTX_OFF = "DTX_OFF" # TODO: Clarify if this should also be tested with DTX, see #1122 TESTCASES = [ # stereo - ["stereo", 32000, "EXT", "DTX_OFF"], + ["stereo", 32000, "STEREO", "DTX_OFF"], ["stereo", 48000, "MONO", "DTX_OFF"], ["stereo", 16400, "5_1", "DTX_OFF"], ["stereo", 256000, "7_1_4", "DTX_OFF"], @@ -42,21 +42,21 @@ TESTCASES = [ # McMasa ["MC_5_1", 16400, "BINAURAL_ROOM_IR", "DTX_OFF"], ["MC_7_1_4", 80000, "mono", "DTX_OFF"], - ["MC_5_1_2", 24400, "EXT", "DTX_OFF"], + ["MC_5_1_2", 24400, "5_1_2", "DTX_OFF"], # paramMC ["MC_5_1_2", 48000, "BINAURAL", "DTX_OFF"], - ["MC_7_1", 80000, "EXT", "DTX_OFF"], + ["MC_7_1", 80000, "7_1", "DTX_OFF"], ["MC_7_1_4", 128000, "FOA", "DTX_OFF"], # paramUpmix ["MC_7_1_4", 160000, "stereo", "DTX_OFF"], # discrete MC ["MC_5_1_2", 512000, "BINAURAL_ROOM_REVERB", "DTX_OFF"], - ["MC_7_1", 128000, "EXT", "DTX_OFF"], + ["MC_7_1", 128000, "7_1", "DTX_OFF"], ["MC_7_1_4", 256000, "5_1", "DTX_OFF"], # SBA ["HOA3", 64000, "BINAURAL", "DTX_OFF"], ["FOA", 256000, "BINAURAL_ROOM_REVERB", "DTX_OFF"], - ["HOA2", 80000, "EXT", "DTX_OFF"], + ["HOA2", 80000, "HOA2", "DTX_OFF"], ["HOA3", 512000, "5_1_4", "DTX_OFF"], ["FOA", 80000, "stereo", "DTX_OFF"], # OMASA @@ -64,11 +64,11 @@ TESTCASES = [ ["OMASA_ISM2", 24400, "MONO", "DTX_OFF"], ["OMASA_ISM3", 80000, "7_1_4", "DTX_OFF"], ["OMASA_ISM4", 64000, "HOA3", "DTX_OFF"], - ["OMASA_ISM2", 32000, "EXT", "DTX_OFF"], + # ["OMASA_ISM2", 32000, "EXT", "DTX_OFF"], # OSBA ["OSBA_ISM2_HOA2", 64000, "BINAURAL_ROOM_IR", "DTX_OFF"], - ["OSBA_ISM4_FOA", 512000, "BINAURAL_ROOM_REVERB", "DTX_OFF"], - ["OSBA_ISM3_HOA3", 128000, "EXT", "DTX_OFF"], + # ["OSBA_ISM4_FOA", 512000, "BINAURAL_ROOM_REVERB", "DTX_OFF"], + # ["OSBA_ISM3_HOA3", 128000, "EXT", "DTX_OFF"], ["OSBA_ISM2_HOA3", 96000, "5_1", "DTX_OFF"], ["OSBA_ISM1_HOA2", 32000, "mono", "DTX_OFF"], ] -- GitLab From 0e2f17954e99be323c05d86b1b7de2ad025651a6 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 13 Dec 2024 12:26:47 +0100 Subject: [PATCH 02/46] fix ltv basop encoder .prm file --- .../config/self_test_ltv_basop_encoder.prm | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/scripts/config/self_test_ltv_basop_encoder.prm b/scripts/config/self_test_ltv_basop_encoder.prm index 66f96903d2..22cec5b67e 100644 --- a/scripts/config/self_test_ltv_basop_encoder.prm +++ b/scripts/config/self_test_ltv_basop_encoder.prm @@ -320,35 +320,35 @@ ../IVAS_dec EXT 48 bit testv/ltvOSBA_4ISM_p3OA48c.wav_EXT_512000_48-48.tst // OSBA planar 2OA 4ISM at 512 kbps, 48 kHz in, 48 kHz out, EXT out -../IVAS_cod -ism_sba 4 -2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 512000 48 testv/ltv48_OSBA_4ISM_HOA2.wav bit +../IVAS_cod -ism_sba 4 -2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv testv/ltv48_OSBA_3ISM_HOA2_ISM3.csv testv/ltv48_OSBA_4ISM_HOA2_ISM4.csv 512000 48 testv/ltv48_OSBA_4ISM_HOA2.wav bit ../IVAS_dec EXT 48 bit testv/ltvOSBA_4ISM_p3OA48c.wav_EXT_512000_48-48.tst // OSBA FOA 4ISM at 512 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 512000 48 testv/ltv48_OSBA_4ISM_FOA.wav bit +../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv testv/ltv48_OSBA_2ISM_FOA_ISM2.csv testv/ltv48_OSBA_3ISM_FOA_ISM3.csv testv/ltv48_OSBA_4ISM_FOA_ISM4.csv 512000 48 testv/ltv48_OSBA_4ISM_FOA.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_4ISM_FOA.wav_EXT_512000_48-48.tst // OSBA FOA 4ISM at 512 kbps, 32kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 512000 32 testv/ltv32_OSBA_4ISM_FOA.wav bit +../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv testv/ltv48_OSBA_2ISM_FOA_ISM2.csv testv/ltv48_OSBA_3ISM_FOA_ISM3.csv testv/ltv48_OSBA_4ISM_FOA_ISM4.csv 512000 32 testv/ltv32_OSBA_4ISM_FOA.wav bit ../IVAS_dec EXT 48 bit testv/ltv32_OSBA_4ISM_FOA.wav_EXT_512000_32-48.tst // OSBA FOA 4ISM at 384 kbps, 32kHz in, 32kHz out, EXT out -../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 384000 32 testv/ltv32_OSBA_4ISM_FOA.wav bit +../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv testv/ltv48_OSBA_2ISM_FOA_ISM2.csv testv/ltv48_OSBA_3ISM_FOA_ISM3.csv testv/ltv48_OSBA_4ISM_FOA_ISM4.csv 384000 32 testv/ltv32_OSBA_4ISM_FOA.wav bit ../IVAS_dec EXT 32 bit testv/ltv32_OSBA_4ISM_FOA.wav_EXT_384000_32-32.tst // OSBA FOA 4ISM at 256 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 256000 48 testv/ltv48_OSBA_4ISM_FOA.wav bit +../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv testv/ltv48_OSBA_2ISM_FOA_ISM2.csv testv/ltv48_OSBA_3ISM_FOA_ISM3.csv testv/ltv48_OSBA_4ISM_FOA_ISM4.csv 256000 48 testv/ltv48_OSBA_4ISM_FOA.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_4ISM_FOA.wav_EXT_256000_48-48.tst // OSBA FOA 3ISM at 128 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 3 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv 128000 48 testv/ltv48_OSBA_3ISM_FOA.wav bit +../IVAS_cod -ism_sba 3 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv testv/ltv48_OSBA_2ISM_FOA_ISM2.csv testv/ltv48_OSBA_3ISM_FOA_ISM3.csv 128000 48 testv/ltv48_OSBA_3ISM_FOA.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_3ISM_FOA.wav_EXT_128000_48-48.tst // OSBA FOA 1ISM at 48 kbps, 16kHz in, 16kHz out, EXT out -../IVAS_cod -ism_sba 1 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv 32000 16 testv/ltv16_OSBA_1ISM_FOA.wav bit +../IVAS_cod -ism_sba 1 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv 32000 16 testv/ltv16_OSBA_1ISM_FOA.wav bit ../IVAS_dec EXT 16 bit testv/ltv16_OSBA_1ISM_FOA.wav_EXT_32000_16-16.tst // OSBA FOA 1ISM at 32 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 1 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv 32000 48 testv/ltv48_OSBA_1ISM_FOA.wav bit +../IVAS_cod -ism_sba 1 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv 32000 48 testv/ltv48_OSBA_1ISM_FOA.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_1ISM_FOA.wav_EXT_32000_48-48.tst // OSBA 3OA 4ISM bitrate switching 16.4 to 512, 48kHz in, 48kHz out, EXT out @@ -405,32 +405,32 @@ ../IVAS_dec EXT 32 bit testv/ltv48_OSBA_3ISM_HOA3.wav_EXT_sw_48-32.tst // OSBA 3ISM 2OA at bitrate switching 13.2 to 512 kbps, 48kHz in, 32kHz out, EXT out -../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv ../scripts/switchPaths/sw_13k2_512k.bin 48 testv/ltv48_OSBA_3ISM_HOA2.wav bit +../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv testv/ltv48_OSBA_3ISM_HOA2_ISM3.csv ../scripts/switchPaths/sw_13k2_512k.bin 48 testv/ltv48_OSBA_3ISM_HOA2.wav bit ../IVAS_dec EXT 32 bit testv/ltv48_OSBA_3ISM_HOA2.wav_EXT_sw_48-32.tst // OSBA 2OA 4ISM at 384 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 4 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 384000 48 testv/ltv48_OSBA_4ISM_HOA2.wav bit +../IVAS_cod -ism_sba 4 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv testv/ltv48_OSBA_3ISM_HOA2_ISM3.csv testv/ltv48_OSBA_4ISM_HOA2_ISM4.csv 384000 48 testv/ltv48_OSBA_4ISM_HOA2.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_4ISM_HOA2.wav_EXT_384000_48-48.tst // OSBA 2OA 3ISM at 96 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv 96000 48 testv/ltv48_OSBA_3ISM_HOA2.wav bit +../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv testv/ltv48_OSBA_3ISM_HOA2_ISM3.csv 96000 48 testv/ltv48_OSBA_3ISM_HOA2.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_3ISM_HOA2.wav_EXT_96000_48-48.tst // OSBA 2OA 3ISM at 384 kbps, 16kHz in, 16kHz out, EXT out -../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv 384000 16 testv/ltv16_OSBA_3ISM_HOA2.wav bit +../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv testv/ltv48_OSBA_3ISM_HOA2_ISM3.csv 384000 16 testv/ltv16_OSBA_3ISM_HOA2.wav bit ../IVAS_dec EXT 16 bit testv/ltv16_OSBA_3ISM_HOA2.wav_EXT56000_16-16.tst // OSBA 2OA 2ISM at 64 kbps, 32kHz in, 16kHz out, EXT out -../IVAS_cod -ism_sba 2 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv 48000 32 testv/ltv32_OSBA_2ISM_HOA2.wav bit +../IVAS_cod -ism_sba 2 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv 48000 32 testv/ltv32_OSBA_2ISM_HOA2.wav bit ../IVAS_dec EXT 16 bit testv/ltv32_OSBA_2ISM_HOA2.wav_EXT_64000_32-16.tst // OSBA 2OA 2ISM at 48 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 2 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv 48000 48 testv/ltv48_OSBA_2ISM_HOA2.wav bit +../IVAS_cod -ism_sba 2 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv 48000 48 testv/ltv48_OSBA_2ISM_HOA2.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_2ISM_HOA2.wav_EXT_48000_48-48.tst // OSBA 2OA 1ISM at 24.4 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 1 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv 24400 48 testv/ltv48_OSBA_1ISM_EXT.wav bit -../IVAS_dec EXT 48 bit testv/ltv48_OSBA_1ISM_EXT.wav_EXT_24400_48-48.tst +../IVAS_cod -ism_sba 1 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv 24400 48 testv/ltv48_OSBA_1ISM_HOA2.wav bit +../IVAS_dec EXT 48 bit testv/ltv48_OSBA_1ISM_HOA2.wav_EXT_24400_48-48.tst // OMASA 2TC 4ISM at br sw techs 13.2 to 512 kbps start 384 kbps, 48kHz in, 48kHz out, BINAURAL out ../IVAS_cod -ism_masa 4 2 testv/ltv48_OMASA_1ISM_2TC_ISM1.csv testv/ltv48_OMASA_2ISM_2TC_ISM2.csv testv/ltv48_OMASA_3ISM_2TC_ISM3.csv testv/ltv48_OMASA_4ISM_2TC_ISM4.csv testv/ltv48_OMASA_4ISM_2TC.met ../scripts/switchPaths/sw_13k2_512k_2fr_start_384k_omasatechs_4ism.bin 48 testv/ltv48_OMASA_4ISM_2TC.wav bit -- GitLab From 6e3204d94a77e84b0632b5cfac90d49622997ff2 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 13 Dec 2024 14:23:34 +0100 Subject: [PATCH 03/46] comment out problematic testcase there is an issue with ivas-float-update that needs to be fixed first --- scripts/config/self_test_ltv_basop_encoder.prm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/config/self_test_ltv_basop_encoder.prm b/scripts/config/self_test_ltv_basop_encoder.prm index 22cec5b67e..ca20519cb2 100644 --- a/scripts/config/self_test_ltv_basop_encoder.prm +++ b/scripts/config/self_test_ltv_basop_encoder.prm @@ -276,8 +276,8 @@ ../IVAS_dec HOA3 48 bit testv/ltv48_HOA3.wav_sw_48-48_HOA3.tst // SBA 3OA bitrate switching from 13.2 kbps to 128 kbps, 48kHz in, 48kHz out, DTX on, HOA3 out -../IVAS_cod -dtx -sba 3 ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 48 testv/ltv48_HOA3.wav bit -../IVAS_dec HOA3 48 bit testv/ltv48_HOA3.wav_sw_48-48_DTX_HOA3.tst +//../IVAS_cod -dtx -sba 3 ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 48 testv/ltv48_HOA3.wav bit +//../IVAS_dec HOA3 48 bit testv/ltv48_HOA3.wav_sw_48-48_DTX_HOA3.tst // SBA 3OA bitrate switching from 13.2 kbps to 128 kbps, 32kHz in, 32kHz out, DTX on, HOA3 out ../IVAS_cod -dtx -sba 3 ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 32 testv/ltv32_HOA3.wav bit -- GitLab From 65c6d89216307ec1728657a883c7a33f922d9027 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 8 Jan 2025 16:49:41 +0100 Subject: [PATCH 04/46] add parsing of command lines into the check_for_changes script --- .../basop_check_for_changes_in_testcases.py | 87 ++++++++++++++++++- 1 file changed, 83 insertions(+), 4 deletions(-) diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index e4116cb76d..0942bbb537 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -35,6 +35,9 @@ import argparse import sys import os import pathlib +import re +import xml.etree.ElementTree as ET +from typing import Tuple # set positive threshold for "lower is better" metrics, negative for "higher is better" @@ -48,8 +51,12 @@ COLS_2_THRESHOLDS = { OUTFILE_CRASHES = "changes_crashes.csv" OUTFILE_SCORES = "changes_{}.csv" +PATTERN_ENC = r"... encoder command:\s*(.*)\s*... encoder stdout:" +PATTERN_DEC = r"... decoder command:\s*(.*)\s*... decoder stdout:" + def main(args): + xml_report = args.xml_report df_curr = pd.read_csv(args.csv_current, sep=";") df_prev = pd.read_csv(args.csv_previous, sep=";") df_merged = pd.merge(df_curr, df_prev, on="testcase", suffixes=["-curr", "-prev"]) @@ -77,7 +84,6 @@ def main(args): df_crashes_introduced = df_merged[mask_crash_introduced][display_cols].reset_index( drop=True ) - df_crashes_introduced.to_csv(OUTFILE_CRASHES, sep=";") if sum(mask_crash_introduced) > 0: regressions_found = True @@ -85,15 +91,32 @@ def main(args): print(df_crashes_introduced) print() + if xml_report is not None: + cmdlines_crashes_introduced = get_command_lines_for_testcases( + df_crashes_introduced["testcase"], xml_report + ) + df_crashes_introduced = pd.merge( + df_crashes_introduced, cmdlines_crashes_introduced, on="testcase" + ) + df_crashes_introduced.to_csv(OUTFILE_CRASHES, sep=";") + if args.show_improvements and sum(mask_crash_fixed) > 0: df_crashes_fixed = df_merged[mask_crash_fixed][display_cols].reset_index( drop=True ) - df_crashes_fixed.to_csv(OUTFILE_CRASHES, mode="a", sep=";") print("---------------Testcases that fixed crashes---------------") print(df_crashes_fixed) print() + if xml_report is not None: + cmdlines_crashes_fixed = get_command_lines_for_testcases( + df_crashes_fixed["testcase"], xml_report + ) + df_crashes_fixed = pd.merge( + df_crashes_fixed, cmdlines_crashes_fixed, on="testcase" + ) + df_crashes_fixed.to_csv(OUTFILE_CRASHES, mode="a", sep=";") + # remove columns with ERRORs in any of the csv files before comparing the numerical columns mask_no_errors = (df_merged[col_curr] != "ERROR") & (df_merged[col_prev] != "ERROR") df_merged = df_merged[mask_no_errors].reset_index(drop=True) @@ -115,7 +138,6 @@ def main(args): display_cols = ["testcase", col_curr, col_prev, col_diff] outfile = OUTFILE_SCORES.format(col.replace(" ", "_")) df_worse = df_merged[mask_worse][display_cols].reset_index(drop=True) - df_worse.to_csv(outfile, sep=";") if sum(mask_worse) > 0: regressions_found = True print( @@ -124,18 +146,70 @@ def main(args): print(df_worse) print() + if xml_report is not None: + cmdlines_worse = get_command_lines_for_testcases( + df_worse["testcase"], xml_report + ) + df_worse = pd.merge(df_worse, cmdlines_worse, on="testcase") + df_worse.to_csv(outfile, sep=";") + if args.show_improvements and sum(mask_better) > 0: df_better = df_merged[mask_better][display_cols].reset_index(drop=True) - df_better.to_csv(outfile, mode="a", sep=";") print( f"---------------Testcases that got better wrt to {col}---------------" ) print(df_better) print() + if xml_report is not None: + cmdlines_better = get_command_lines_for_testcases( + df_better["testcase"], xml_report + ) + df_better = pd.merge(df_better, cmdlines_better, on="testcase") + df_better.to_csv(outfile, mode="a", sep=";") + return int(regressions_found) +def get_command_lines_for_testcases( + testcases: pd.Series, xml_report: pathlib.Path +) -> pd.DataFrame: + testcase_elems = [ + e + for _, e in ET.iterparse(xml_report) + if e.tag == "testcase" and e.attrib["name"] in testcases.values + ] + + cmdlines = {"testcase": [], "enc_cmd": [], "dec_cmd": []} + for elem in testcase_elems: + testcase_name = elem.attrib["name"] + enc_cmd = "" + dec_cmd = "" + if (system_out := elem.find("system-out")) is not None: + enc_cmd, dec_cmd = extract_cmdlines(system_out.text) + + cmdlines["testcase"].append(testcase_name) + cmdlines["enc_cmd"].append(enc_cmd) + cmdlines["dec_cmd"].append(dec_cmd) + + return pd.DataFrame(cmdlines) + + +def extract_cmdlines(text: str) -> Tuple[str, str]: + enc_cmdline = "" + dec_cmdline = "" + + match_enc = re.search(PATTERN_ENC, text) + match_dec = re.search(PATTERN_DEC, text) + if match_enc is not None and match_dec is not None: + enc_cmdline = match_enc.group(1) + dec_cmdline = match_dec.group(1) + + # TODO: post-process paths + + return enc_cmdline, dec_cmdline + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("csv_current") @@ -147,6 +221,11 @@ if __name__ == "__main__": default=COLS_2_THRESHOLDS.keys(), ) parser.add_argument("--show_improvements", action="store_true") + parser.add_argument( + "--xml_report", + help="XMLxml_report report file from pytest run. Pass to add command lines to the output files.", + default=None, + ) args = parser.parse_args() sys.exit(main(args)) -- GitLab From 36824cdea534df54cc4115ec214b8a03614ac7dc Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 8 Jan 2025 17:23:59 +0100 Subject: [PATCH 05/46] normalize paths in command lines --- .../basop_check_for_changes_in_testcases.py | 48 +++++++++++++++---- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index 0942bbb537..5d904fee2d 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -93,7 +93,7 @@ def main(args): if xml_report is not None: cmdlines_crashes_introduced = get_command_lines_for_testcases( - df_crashes_introduced["testcase"], xml_report + df_crashes_introduced["testcase"], xml_report, args.inject_cwd ) df_crashes_introduced = pd.merge( df_crashes_introduced, cmdlines_crashes_introduced, on="testcase" @@ -110,7 +110,7 @@ def main(args): if xml_report is not None: cmdlines_crashes_fixed = get_command_lines_for_testcases( - df_crashes_fixed["testcase"], xml_report + df_crashes_fixed["testcase"], xml_report, args.inject_cwd ) df_crashes_fixed = pd.merge( df_crashes_fixed, cmdlines_crashes_fixed, on="testcase" @@ -148,7 +148,7 @@ def main(args): if xml_report is not None: cmdlines_worse = get_command_lines_for_testcases( - df_worse["testcase"], xml_report + df_worse["testcase"], xml_report, args.inject_cwd ) df_worse = pd.merge(df_worse, cmdlines_worse, on="testcase") df_worse.to_csv(outfile, sep=";") @@ -163,7 +163,7 @@ def main(args): if xml_report is not None: cmdlines_better = get_command_lines_for_testcases( - df_better["testcase"], xml_report + df_better["testcase"], xml_report, args.inject_cwd ) df_better = pd.merge(df_better, cmdlines_better, on="testcase") df_better.to_csv(outfile, mode="a", sep=";") @@ -172,7 +172,7 @@ def main(args): def get_command_lines_for_testcases( - testcases: pd.Series, xml_report: pathlib.Path + testcases: pd.Series, xml_report: pathlib.Path, cwd: pathlib.Path ) -> pd.DataFrame: testcase_elems = [ e @@ -186,7 +186,7 @@ def get_command_lines_for_testcases( enc_cmd = "" dec_cmd = "" if (system_out := elem.find("system-out")) is not None: - enc_cmd, dec_cmd = extract_cmdlines(system_out.text) + enc_cmd, dec_cmd = extract_cmdlines(system_out.text, cwd) cmdlines["testcase"].append(testcase_name) cmdlines["enc_cmd"].append(enc_cmd) @@ -195,7 +195,7 @@ def get_command_lines_for_testcases( return pd.DataFrame(cmdlines) -def extract_cmdlines(text: str) -> Tuple[str, str]: +def extract_cmdlines(text: str, cwd: pathlib.Path) -> Tuple[str, str]: enc_cmdline = "" dec_cmdline = "" @@ -205,11 +205,37 @@ def extract_cmdlines(text: str) -> Tuple[str, str]: enc_cmdline = match_enc.group(1) dec_cmdline = match_dec.group(1) - # TODO: post-process paths + enc_cmdline = postprocess_cmdline(enc_cmdline, cwd) + dec_cmdline = postprocess_cmdline(dec_cmdline, cwd) return enc_cmdline, dec_cmdline +def postprocess_cmdline(cmdline: str, cwd: pathlib.Path) -> str: + cmdline_split = cmdline.split() + cmdline_proc = [] + + # change absolute paths into relative ones + # remove the "quite" flag + # for output and bitstream files only keep the filename + for elem in cmdline_split: + print(elem) + if elem == "-q": + continue + elif (elem_as_path := pathlib.Path(elem)).is_absolute(): + if elem_as_path.suffix == ".192" or ( + elem_as_path.suffix == ".wav" + and cmdline_split.index(elem) == len(cmdline_split) - 1 + ): + cmdline_proc.append(elem_as_path.name) + else: + cmdline_proc.append(str(elem_as_path.relative_to(cwd))) + else: + cmdline_proc.append(elem) + + return " ".join(cmdline_proc) + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("csv_current") @@ -226,6 +252,12 @@ if __name__ == "__main__": help="XMLxml_report report file from pytest run. Pass to add command lines to the output files.", default=None, ) + parser.add_argument( + "--inject_cwd", + help="Use this as cwd when pruning the long paths in the command lines. Debug option for testing.", + default=pathlib.Path(__file__).parent.absolute(), + type=pathlib.Path, + ) args = parser.parse_args() sys.exit(main(args)) -- GitLab From 366f249569a6ed229e174a6090fe36a532399702 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 9 Jan 2025 11:29:59 +0100 Subject: [PATCH 06/46] remove leftover print from debugging --- scripts/basop_check_for_changes_in_testcases.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index 5d904fee2d..ed67aa279d 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -219,7 +219,6 @@ def postprocess_cmdline(cmdline: str, cwd: pathlib.Path) -> str: # remove the "quite" flag # for output and bitstream files only keep the filename for elem in cmdline_split: - print(elem) if elem == "-q": continue elif (elem_as_path := pathlib.Path(elem)).is_absolute(): -- GitLab From 0696cb4a9e3ca61cc7347f369fc69b4bae9f1f05 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 13 Jan 2025 17:11:33 +0100 Subject: [PATCH 07/46] add capturing of eid-xor and network sim cmdlines --- .../basop_check_for_changes_in_testcases.py | 46 +++++++++++++------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index ed67aa279d..c7c2e0f785 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -53,6 +53,10 @@ OUTFILE_SCORES = "changes_{}.csv" PATTERN_ENC = r"... encoder command:\s*(.*)\s*... encoder stdout:" PATTERN_DEC = r"... decoder command:\s*(.*)\s*... decoder stdout:" +PATTERN_EID = r"eid-xor command:\s*(.*)\s*" +PATTERN_NETSIM = r"netsim command:\s*(.*)\s*" + +PATTERNS = [PATTERN_ENC, PATTERN_DEC, PATTERN_EID, PATTERN_NETSIM] def main(args): @@ -180,35 +184,47 @@ def get_command_lines_for_testcases( if e.tag == "testcase" and e.attrib["name"] in testcases.values ] - cmdlines = {"testcase": [], "enc_cmd": [], "dec_cmd": []} + cmdlines = { + "testcase": [], + "enc_cmd": [], + "dec_cmd": [], + "eid-xor_cmd": [], + "netsim_cmd": [], + } for elem in testcase_elems: testcase_name = elem.attrib["name"] enc_cmd = "" dec_cmd = "" + eid_cmd = "" + netsim_cmd = "" if (system_out := elem.find("system-out")) is not None: - enc_cmd, dec_cmd = extract_cmdlines(system_out.text, cwd) + ( + enc_cmd, + dec_cmd, + eid_cmd, + netsim_cmd, + ) = extract_cmdlines(system_out.text, cwd) cmdlines["testcase"].append(testcase_name) cmdlines["enc_cmd"].append(enc_cmd) cmdlines["dec_cmd"].append(dec_cmd) + cmdlines["eid-xor_cmd"].append(eid_cmd) + cmdlines["netsim_cmd"].append(netsim_cmd) return pd.DataFrame(cmdlines) -def extract_cmdlines(text: str, cwd: pathlib.Path) -> Tuple[str, str]: - enc_cmdline = "" - dec_cmdline = "" - - match_enc = re.search(PATTERN_ENC, text) - match_dec = re.search(PATTERN_DEC, text) - if match_enc is not None and match_dec is not None: - enc_cmdline = match_enc.group(1) - dec_cmdline = match_dec.group(1) - - enc_cmdline = postprocess_cmdline(enc_cmdline, cwd) - dec_cmdline = postprocess_cmdline(dec_cmdline, cwd) +def extract_cmdlines(text: str, cwd: pathlib.Path) -> list[str]: + cmdlines = [] + for p in PATTERNS: + m = re.search(p, text) + if m is not None: + cmdline = postprocess_cmdline(m.group(1), cwd) + cmdlines.append(cmdline) + else: + cmdlines.append("") - return enc_cmdline, dec_cmdline + return cmdlines def postprocess_cmdline(cmdline: str, cwd: pathlib.Path) -> str: -- GitLab From d004971f25e2ad2fd9ad0731450a4da55c5d921d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 21 Jan 2025 13:15:04 +0100 Subject: [PATCH 08/46] correct default value for cwd in basop_check_for_changes_in_testcases.py --- scripts/basop_check_for_changes_in_testcases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index c7c2e0f785..b47f0d6cc2 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -270,7 +270,7 @@ if __name__ == "__main__": parser.add_argument( "--inject_cwd", help="Use this as cwd when pruning the long paths in the command lines. Debug option for testing.", - default=pathlib.Path(__file__).parent.absolute(), + default=pathlib.Path(os.getcwd()).absolute(), type=pathlib.Path, ) -- GitLab From a5c4bb901bcb5f7a047746c31e910f571371d14e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=BCller?= Date: Wed, 29 Jan 2025 12:12:01 +0100 Subject: [PATCH 09/46] Add script to filter out smoke test items broken in basop --- scripts/smoketest-basop-filter.py | 53 +++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 scripts/smoketest-basop-filter.py diff --git a/scripts/smoketest-basop-filter.py b/scripts/smoketest-basop-filter.py new file mode 100644 index 0000000000..faa170f7e0 --- /dev/null +++ b/scripts/smoketest-basop-filter.py @@ -0,0 +1,53 @@ +#! /usr/bin/env python3 + +import argparse +import json +import logging +import sys + + +logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') +logger = logging.getLogger() + + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument("input", help="input JSON", type=argparse.FileType('r')) + parser.add_argument("output", help="output JSON", type=argparse.FileType('w'), nargs="?") + parser.add_argument("--inline", "-i", help="patch input inline", action="store_true") + + args = parser.parse_args() + + if not args.inline and not args.output: + logging.error("You must specify an output file or use --inline") + return 1 + + modes = json.load(args.input) + + for mode in ("MC", "stereo", "OMASA"): + for encoder_config_name in modes[mode].keys(): + try: + del modes[mode][encoder_config_name]["dec"]["EXT"] + except KeyError: + logger.warning(f"No EXT dec in {mode} -> {encoder_config_name}, skipping") + continue + else: + logger.info(f"Found EXT dec in {mode} -> {encoder_config_name}, removing") + + for encoder_config_name in tuple(modes["OSBA"].keys()): + if not "FOA" in encoder_config_name: + continue + + logger.info(f"Removing OSBA FOA encoder config {encoder_config_name}") + del modes["OSBA"][encoder_config_name] + + if args.inline: + with open(args.input.name, "w") as f: + json.dump(modes, f, indent=4) + else: + json.dump(modes, args.output, indent=4) + + +if __name__ == "__main__": + sys.exit(main()) -- GitLab From 79e6dbad08720b20202745888eb52be09e8ec4f0 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Tue, 28 Jan 2025 11:23:09 +0100 Subject: [PATCH 10/46] [fix] missing files for conformance test --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9d3a624b20..416b86f3fc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1460,6 +1460,7 @@ ivas-conformance: - mkdir testvec/bin - cp -force -ErrorAction Ignore scripts/testv/* testvec/testv - cp -r -force -ErrorAction Ignore scripts/ls_layouts testvec + - cp -r -force -ErrorAction Ignore scripts/object_edit testvec - cp -r -force -ErrorAction Ignore scripts/switchPaths testvec - cp -r -force -ErrorAction Ignore scripts/trajectories testvec - cp -r -force -ErrorAction Ignore scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface @@ -1546,6 +1547,7 @@ ivas-conformance-linux: - mkdir testvec/bin - cp -r scripts/testv/* testvec/testv - cp -r scripts/ls_layouts testvec + - cp -r scripts/object_edit testvec - cp -r scripts/switchPaths testvec - cp -r scripts/trajectories testvec - cp -r scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface -- GitLab From 7ba0706d6efb311df72a22dcf31bd961d377417d Mon Sep 17 00:00:00 2001 From: Lauros Pajunen Date: Tue, 28 Jan 2025 15:04:31 +0200 Subject: [PATCH 11/46] Update search keyword for WMOPS total line in logs --- scripts/pyivastest/IvasModeAnalyzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pyivastest/IvasModeAnalyzer.py b/scripts/pyivastest/IvasModeAnalyzer.py index 8d1dd2ddf0..91bf0b44ae 100644 --- a/scripts/pyivastest/IvasModeAnalyzer.py +++ b/scripts/pyivastest/IvasModeAnalyzer.py @@ -43,7 +43,7 @@ from pyivastest.IvasModeCollector import IvasModeCollector INSTRUMENTED_RESULTS = { "WMOPS": { - "keyword": "total", + "keyword": " total ", "number_format": "{:.5g}", "position": 2, "max_or_add": "add", -- GitLab From 426e2c174bbbb670d177b1c37c8d09c4680376c1 Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Wed, 29 Jan 2025 17:12:48 +0100 Subject: [PATCH 12/46] fix if() condition, prevent crash when CPE is NULL --- lib_enc/ivas_core_pre_proc_front.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib_enc/ivas_core_pre_proc_front.c b/lib_enc/ivas_core_pre_proc_front.c index 36da36c128..4d43fce471 100644 --- a/lib_enc/ivas_core_pre_proc_front.c +++ b/lib_enc/ivas_core_pre_proc_front.c @@ -222,8 +222,9 @@ ivas_error pre_proc_front_ivas( } #ifdef DEBUG_MODE_INFO - if ( !( hCPE->hStereoTD != NULL && n > 0 ) ) + if ( !( hCPE != NULL && hCPE->hStereoTD != NULL && n > 0 ) ) { + /* for TD stereo only write out first channel. The existence of a second channel can vary, this is just easier to handle */ int16_t tmp_dmx_in[L_FRAME48k]; mvr2s( signal_in - NS2SA( st->input_Fs, ACELP_LOOK_NS ), tmp_dmx_in, input_frame ); dbgwrite( tmp_dmx_in, sizeof( int16_t ), input_frame, 1, strcat( fname( debug_dir, "ivas_input_dmx", 0, n + 1, ENC ), ".pcm" ) ); -- GitLab From 283001f6bbd21dbd29dc785ef8e56f0d5d2953d6 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 30 Jan 2025 13:14:03 +0100 Subject: [PATCH 13/46] update remove_unsupported_testcases.py --- ci/remove_unsupported_testcases.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/ci/remove_unsupported_testcases.py b/ci/remove_unsupported_testcases.py index b316a417e5..a1995b6d04 100644 --- a/ci/remove_unsupported_testcases.py +++ b/ci/remove_unsupported_testcases.py @@ -66,10 +66,28 @@ TESTCASES = [ "MASA 1TC at 256kbps, 48kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out custom configuration", "MASA 1TC at 256 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out, HR custom configuration", "OMASA 2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out", + # object editing starts here + "OMASA 2Dir2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM", + "OMASA 2Dir2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC", + "OMASA 2Dir2TC 2ISM at 96 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC", + "OMASA 2Dir2TC 2ISM br sw techs 13.2 to 512 kbps start 48 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing", + # the next four lines are basically duplicates of the last four, but without "Dir", because the ltv MASA cases do not have dedicated testvectors for numbers of directions + "OMASA 2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM", + "OMASA 2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC", + "OMASA 2TC 2ISM at 96 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC", + "OMASA 2TC 2ISM br sw techs 13.2 to 512 kbps start 48 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing", + "OSBA 3OA 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, DISC", + "OSBA 2OA 3ISM at 128 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC", + "OSBA 2OA 3ISM at bitrate switching 13.2 to 512 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing", + "4 ISM with metadata at 32 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, PARAM_ISM", + "4 ISM with metadata at 128 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, DISC", + "3 ISM with metadata at 24.4 kbps, 48 kHz in, 48 kHz out, FOA out, object editing, JBM Prof 5, PARAM_ISM", + "3 ISM with metadata at 384 kbps, 48 kHz in, 48 kHz out, FOA out, object editing, JBM Prof 5, DISC", + "4 ISM with metadata bitrate switching from 32 kbps to 48 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_IR out, object editing", + "4 ISM with metadata bitrate switching from 48 kbps to 32 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, JBM Prof 5", ] - def remove_testcases(cfg: Path, testcases: list): """ Go through file line by line and copy all testcases except the given ones -- GitLab From c05306e832004a095fae908910a3ed9ae0b77385 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 31 Jan 2025 15:48:24 +0100 Subject: [PATCH 14/46] run smoke test with filtered config in MRs to basop-ci-branch --- .gitlab-ci.yml | 101 ++++++++++++++++++++++++++++++------------------- 1 file changed, 63 insertions(+), 38 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 416b86f3fc..6d0d8bf458 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -132,6 +132,12 @@ stages: - sed -i.bak -e "s/\/\*\ *\(#define\ *DEBUGGING\ *\)\*\//\1/g" lib_com/options.h - sed -i.bak -e "s/\/\/\ *\(#define\ *DEBUGGING\ *\)/\1/g" lib_com/options.h +.get-basop-float-reference: &get-basop-float-reference + - git clone -b $BASOP_REFERENCE_BRANCH https://forge.3gpp.org/rep/sa4/audio/ivas-basop.git --single-branch + - make -j -C ivas-basop + - cp ivas-basop/IVAS_cod ./ + - cp ivas-basop/IVAS_dec ./ + .merge-request-comparison-setup-codec: &merge-request-comparison-setup-codec ### build test binaries, initial clean for paranoia reasons - *disable-debugging-macro @@ -279,10 +285,24 @@ stages: when: never - when: on_success -.rules-merge-request: +.rules-merge-request-to-main: + extends: .rules-basis + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + +.rules-merge-request-to-basop-ci-branch: + extends: .rules-basis + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "basop-ci-branch" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + +.rules-merge-request-to-basop-ci-branch-or-main: extends: .rules-basis rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "basop-ci-branch" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main") - if: $CI_PIPELINE_SOURCE == 'push' when: never @@ -327,7 +347,7 @@ stages: stage: test extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week @@ -396,7 +416,7 @@ uninterruptible: branch-is-up-to-date-with-main-pre: extends: - - .rules-merge-request + - .rules-merge-request-to-main stage: prevalidate needs: [] tags: @@ -408,7 +428,7 @@ branch-is-up-to-date-with-main-pre: check-self-test-names-pre: extends: - - .rules-merge-request + - .rules-merge-request-to-main stage: prevalidate needs: [] tags: @@ -418,7 +438,7 @@ check-self-test-names-pre: check-no-duplicates-in-self-tests: extends: - - .rules-merge-request + - .rules-merge-request-to-main stage: prevalidate needs: [] tags: @@ -428,7 +448,7 @@ check-no-duplicates-in-self-tests: branch-is-up-to-date-with-main-post: extends: - - .rules-merge-request + - .rules-merge-request-to-main stage: postvalidate tags: - ivas-linux @@ -440,24 +460,19 @@ branch-is-up-to-date-with-main-post: .basop-ci-branch-compat-template: extends: - .test-job-linux - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "basop-ci-branch" - changes: - - tests/**/* - - scripts/**/* - - if: $CI_PIPELINE_SOURCE == 'push' - when: never + - .rules-merge-request-to-basop-ci-branch tags: - ivas-linux-fast before_script: - - git clone -b $BASOP_REFERENCE_BRANCH https://forge.3gpp.org/rep/sa4/audio/ivas-basop.git --single-branch - - make -j -C ivas-basop - python3 ci/remove_unsupported_testcases.py $PARAM_FILE - python3 tests/create_short_testvectors.py + - *update-ltv-repo - *copy-ltv-files-to-testv-dir - - python3 -m pytest tests/codec_be_on_mr_nonselection --param_file $PARAM_FILE --update_ref 1 --ref_encoder_path ivas-basop/IVAS_cod --ref_decoder_path ivas-basop/IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml || true + - *get-basop-float-reference + + - python3 -m pytest tests/codec_be_on_mr_nonselection --param_file $PARAM_FILE --update_ref 1 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml || true - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 1 ]; then echo "Run errors encountered with $PARAM_FILE !"; exit $EXIT_CODE_FAIL; fi @@ -579,7 +594,7 @@ build-codec-windows-msbuild: codec-smoke-test: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-basop-ci-branch-or-main timeout: "20 minutes" tags: - ivas-linux-fast @@ -589,7 +604,17 @@ codec-smoke-test: - *print-common-info # LTV update needed as ltv ISM metadata files are used - *update-ltv-repo - - bash ci/smoke_test.sh + + # for MRs to basop-ci-branch, we want to test with the BASOP float reference build + # per default, the smoke test builds the current repo again with WMOPS activated to catch unbalanced instrumentation macros + - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" = "basop-ci-branch" ]; then + - python3 scripts/smoketest-basop-filter.py scripts/config/ivas_modes.json --inline + - *get-basop-float-reference + - bash ci/smoke_test.sh coverage + - else + - bash ci/smoke_test.sh + - fi + ### analyze for failures - if ! [ -s smoke_test_output.txt ] || ! [ -s smoke_test_output_jbm.txt ] || ! [ -s smoke_test_output_hrtf.txt ]; then echo "Error in smoke test"; exit 1; fi - ret_val=0 @@ -648,7 +673,7 @@ codec-usan: pytest-compare-20ms-and-5ms-rendering: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-cmake", "build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"] script: @@ -695,7 +720,7 @@ pytest-compare-20ms-and-5ms-rendering: renderer-smoke-test: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-make"] stage: test script: @@ -717,7 +742,7 @@ renderer-smoke-test: renderer-asan: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-cmake"] stage: test script: @@ -741,7 +766,7 @@ renderer-asan: renderer-msan: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-cmake"] stage: test script: @@ -765,7 +790,7 @@ renderer-msan: renderer-usan: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-cmake"] stage: test script: @@ -793,7 +818,7 @@ renderer-usan: renderer-pytest-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-make"] # TODO: set reasonable timeout, will most likely take less timeout: "20 minutes" @@ -845,7 +870,7 @@ renderer-pytest-on-merge-request: split-rendering-smoke-test: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-make"] stage: test script: @@ -866,7 +891,7 @@ split-rendering-smoke-test: lc3-wrapper-unit-test: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-cmake"] stage: test script: @@ -878,7 +903,7 @@ lc3-wrapper-unit-test: split-rendering-pytest-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-make"] # TODO: set reasonable timeout, will most likely take less timeout: "30 minutes" @@ -950,7 +975,7 @@ split-rendering-pytest-on-merge-request: ivas-pytest-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main stage: compare needs: ["build-codec-linux-cmake", "codec-smoke-test"] timeout: "14 minutes" @@ -1004,7 +1029,7 @@ ivas-pytest-on-merge-request: ivas-interop-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-cmake"] timeout: "10 minutes" @@ -1053,7 +1078,7 @@ ivas-interop-on-merge-request: evs-pytest-on-merge-request: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: compare needs: ["build-codec-linux-cmake", "codec-smoke-test"] timeout: "10 minutes" @@ -1105,7 +1130,7 @@ evs-pytest-on-merge-request: voip-be-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-make"] timeout: "10 minutes" @@ -1118,7 +1143,7 @@ voip-be-on-merge-request: clang-format-check: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main variables: ARTIFACT_BASE_NAME: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--formatting-fix" stage: prevalidate @@ -1165,7 +1190,7 @@ clang-format-check: check-first-frame-is-sid: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main tags: - ivas-linux stage: test @@ -1192,7 +1217,7 @@ check-first-frame-is-sid: .lc3plus-ensure-no-code-changes: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: postvalidate needs: [] timeout: "5 minutes" @@ -1207,7 +1232,7 @@ check-first-frame-is-sid: check-bitexactness-hrtf-rom-and-file: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-cmake"] timeout: "5 minutes" @@ -1229,7 +1254,7 @@ check-bitexactness-hrtf-rom-and-file: check-bitexactness-ext-and-transport-format: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-cmake"] timeout: "5 minutes" @@ -1256,7 +1281,7 @@ check-bitexactness-ext-and-transport-format: # check bitexactness to EVS windows binaries be-2-evs-windows: extends: - - .rules-merge-request + - .rules-merge-request-to-main tags: - ivas-windows stage: test -- GitLab From d685256d153a14d65e1c1d2f031609207860bcbb Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 5 Feb 2025 19:26:01 +0100 Subject: [PATCH 15/46] Add explicit trigger of prepare_combined_format_inputs.py --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6d0d8bf458..4f53b30fcd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -466,6 +466,7 @@ branch-is-up-to-date-with-main-post: before_script: - python3 ci/remove_unsupported_testcases.py $PARAM_FILE - python3 tests/create_short_testvectors.py + - python3 scripts/prepare_combined_format_inputs.py - *update-ltv-repo - *copy-ltv-files-to-testv-dir -- GitLab From 9509479a99c904fb2cd68fd99214e66b8fc77350 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 6 Feb 2025 15:45:38 +0100 Subject: [PATCH 16/46] add ubsan suppression file for basop --- scripts/ubsan_basop.supp | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 scripts/ubsan_basop.supp diff --git a/scripts/ubsan_basop.supp b/scripts/ubsan_basop.supp new file mode 100644 index 0000000000..b836305c7f --- /dev/null +++ b/scripts/ubsan_basop.supp @@ -0,0 +1,35 @@ +# From the scheduled sanitizer tests with self_test.py +# lib_com +implicit-integer-sign-change:lib_com/ACcontextMapping_fx.c +implicit-integer-sign-change:lib_com/bitstream_fx.c +implicit-integer-sign-change:lib_com/enh64.c +implicit-signed-integer-truncation:lib_com/enh1632.c +implicit-signed-integer-truncation:lib_com/enhUL32.c +invalid-shift-base:lib_com/enh64.c +invalid-shift-exponent:lib_com/basop32.c +invalid-shift-exponent:lib_com/enh64.c +out-of-bounds-index:lib_com/ifft_rel.c +out-of-bounds-index:lib_com/log2.c +out-of-bounds-index:lib_com/tcx_ltp_fx.c +out-of-bounds-index:lib_com/trans_direct_fx.c +out-of-bounds-index:lib_com/trans_inv_fx.c +signed-integer-overflow:lib_com/enh64.c +# lib_enc +implicit-integer-sign-change:lib_enc/ACcontextMapping_enc_fx.c +implicit-integer-sign-change:lib_enc/bass_psfilter_enc_fx.c +implicit-integer-sign-change:lib_enc/cod4t64_fx.c +implicit-integer-sign-change:lib_enc/core_enc_ol_fx.c +implicit-integer-sign-change:lib_enc/enc_acelp_fx.c +implicit-integer-sign-change:lib_enc/enc_prm_fx.c +implicit-integer-sign-change:lib_enc/hq_classifier_enc_fx.c +implicit-integer-sign-change:lib_enc/inov_enc_fx.c +implicit-integer-sign-change:lib_enc/lead_indexing_fx.c +implicit-integer-sign-change:lib_enc/range_enc_fx.c +implicit-integer-sign-change:lib_enc/tcx_utils_enc_fx.c +out-of-bounds-index:lib_enc/enc_acelp_fx.c +out-of-bounds-index:lib_enc/enc_gain_fx.c +out-of-bounds-index:lib_enc/pit_enc_fx.c +out-of-bounds-index:lib_enc/pitch_ol2_fx.c +out-of-bounds-index:lib_enc/tcx_ltp_enc_fx.c +# lib_dec +out-of-bounds-index:lib_dec/dec_acelp_fx.c -- GitLab From eb249c2fa46027966f125cb6e8dcfaa29c7460a2 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 7 Feb 2025 09:57:46 +0100 Subject: [PATCH 17/46] fix basop usan supp file --- scripts/ubsan_basop.supp | 65 +++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 34 deletions(-) diff --git a/scripts/ubsan_basop.supp b/scripts/ubsan_basop.supp index b836305c7f..bfd00f3573 100644 --- a/scripts/ubsan_basop.supp +++ b/scripts/ubsan_basop.supp @@ -1,35 +1,32 @@ # From the scheduled sanitizer tests with self_test.py -# lib_com -implicit-integer-sign-change:lib_com/ACcontextMapping_fx.c -implicit-integer-sign-change:lib_com/bitstream_fx.c -implicit-integer-sign-change:lib_com/enh64.c -implicit-signed-integer-truncation:lib_com/enh1632.c -implicit-signed-integer-truncation:lib_com/enhUL32.c -invalid-shift-base:lib_com/enh64.c -invalid-shift-exponent:lib_com/basop32.c -invalid-shift-exponent:lib_com/enh64.c -out-of-bounds-index:lib_com/ifft_rel.c -out-of-bounds-index:lib_com/log2.c -out-of-bounds-index:lib_com/tcx_ltp_fx.c -out-of-bounds-index:lib_com/trans_direct_fx.c -out-of-bounds-index:lib_com/trans_inv_fx.c -signed-integer-overflow:lib_com/enh64.c -# lib_enc -implicit-integer-sign-change:lib_enc/ACcontextMapping_enc_fx.c -implicit-integer-sign-change:lib_enc/bass_psfilter_enc_fx.c -implicit-integer-sign-change:lib_enc/cod4t64_fx.c -implicit-integer-sign-change:lib_enc/core_enc_ol_fx.c -implicit-integer-sign-change:lib_enc/enc_acelp_fx.c -implicit-integer-sign-change:lib_enc/enc_prm_fx.c -implicit-integer-sign-change:lib_enc/hq_classifier_enc_fx.c -implicit-integer-sign-change:lib_enc/inov_enc_fx.c -implicit-integer-sign-change:lib_enc/lead_indexing_fx.c -implicit-integer-sign-change:lib_enc/range_enc_fx.c -implicit-integer-sign-change:lib_enc/tcx_utils_enc_fx.c -out-of-bounds-index:lib_enc/enc_acelp_fx.c -out-of-bounds-index:lib_enc/enc_gain_fx.c -out-of-bounds-index:lib_enc/pit_enc_fx.c -out-of-bounds-index:lib_enc/pitch_ol2_fx.c -out-of-bounds-index:lib_enc/tcx_ltp_enc_fx.c -# lib_dec -out-of-bounds-index:lib_dec/dec_acelp_fx.c +implicit-integer-sign-change:ACcontextMapping_fx.c +implicit-integer-sign-change:bitstream_fx.c +implicit-integer-sign-change:enh64.c +implicit-signed-integer-truncation:enh1632.c +implicit-signed-integer-truncation:enhUL32.c +shift-base:enh64.c +shift-exponent:basop32.c +shift-exponent:enh64.c +bounds:ifft_rel.c +bounds:log2.c +bounds:tcx_ltp_fx.c +bounds:trans_direct_fx.c +bounds:trans_inv_fx.c +signed-integer-overflow:enh64.c +implicit-integer-sign-change:ACcontextMapping_enc_fx.c +implicit-integer-sign-change:bass_psfilter_enc_fx.c +implicit-integer-sign-change:cod4t64_fx.c +implicit-integer-sign-change:core_enc_ol_fx.c +implicit-integer-sign-change:enc_acelp_fx.c +implicit-integer-sign-change:enc_prm_fx.c +implicit-integer-sign-change:hq_classifier_enc_fx.c +implicit-integer-sign-change:inov_enc_fx.c +implicit-integer-sign-change:lead_indexing_fx.c +implicit-integer-sign-change:range_enc_fx.c +implicit-integer-sign-change:tcx_utils_enc_fx.c +bounds:enc_acelp_fx.c +bounds:enc_gain_fx.c +bounds:pit_enc_fx.c +bounds:pitch_ol2_fx.c +bounds:tcx_ltp_enc_fx.c +bounds:dec_acelp_fx.c -- GitLab From 3c1d1312a8fe571ec5693ea95978f1fb86540002 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 10 Feb 2025 10:44:09 +0100 Subject: [PATCH 18/46] do not use bash strict mode for getWmops.sh could be fixed also by using "|| retval=$?" on the IvasBuildAndRunChecks call, but better stay save to not loose complexity graphs --- ci/complexity_measurements/getWmops.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/complexity_measurements/getWmops.sh b/ci/complexity_measurements/getWmops.sh index 3f49630530..750c2c299b 100755 --- a/ci/complexity_measurements/getWmops.sh +++ b/ci/complexity_measurements/getWmops.sh @@ -28,8 +28,6 @@ # accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and # the United Nations Convention on Contracts on the International Sales of Goods. -set -euxo pipefail - function usage { echo "Usage: $0 \"ivas-format(s)\" \"output-format(s)\" \"mode{full(default)|mem_only}\" \"repo{float(default)|basop}\" [wmops_ref_logfile]" exit 1 -- GitLab From f06d238c077bcbaeabbe541813129146e1dbfb24 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 10 Feb 2025 14:28:04 +0100 Subject: [PATCH 19/46] add new job that tests BASOP ref compatibility with --compare_enc_dmx --- .gitlab-ci.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f27836fc82..efaf5c7b1b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,6 +31,7 @@ variables: TESTCASE_TIMEOUT_LTV_SANITIZERS: 2400 BASOP_REFERENCE_BRANCH: "ivas-float-update" SCALE_FACTOR: "3.162" + PYTEST_ARGS: "" default: @@ -473,7 +474,7 @@ branch-is-up-to-date-with-main-post: - *get-basop-float-reference - - python3 -m pytest tests/codec_be_on_mr_nonselection --param_file $PARAM_FILE --update_ref 1 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml || true + - python3 -m pytest tests/codec_be_on_mr_nonselection $PYTEST_ARGS --param_file $PARAM_FILE --update_ref 1 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml || true - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 1 ]; then echo "Run errors encountered with $PARAM_FILE !"; exit $EXIT_CODE_FAIL; fi @@ -522,6 +523,16 @@ check-compatibility-with-basop-reference-branch-encoder-ltv: script: - exit 0 +check-compatibility-with-basop-reference-branch-encoder-dmx-comparison: + extends: + - .basop-ci-branch-compat-template + variables: + PARAM_FILE: scripts/config/self_test_basop_encoder.prm + # USING PYTEST_ADDOPTS env var did not work for some reason when testing locally - maybe because this is a custom option + PYTEST_ARGS: "--compare_dmx_enc" + script: + - exit 0 + # --------------------------------------------------------------- # Build jobs # --------------------------------------------------------------- -- GitLab From 3bdb33b5b59743eefcb90b2244d30075974666ec Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 10 Feb 2025 14:29:52 +0100 Subject: [PATCH 20/46] shorten names for convenience in web pipeline view --- .gitlab-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index efaf5c7b1b..3f0283ba1e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -491,7 +491,7 @@ branch-is-up-to-date-with-main-post: junit: - report-junit.xml -check-compatibility-with-basop-reference-branch-stv: +basop-ref-compat-stv: extends: - .basop-ci-branch-compat-template variables: @@ -499,7 +499,7 @@ check-compatibility-with-basop-reference-branch-stv: script: - exit 0 -check-compatibility-with-basop-reference-branch-ltv: +basop-ref-compat-ltv: extends: - .basop-ci-branch-compat-template variables: @@ -507,7 +507,7 @@ check-compatibility-with-basop-reference-branch-ltv: script: - exit 0 -check-compatibility-with-basop-reference-branch-encoder-stv: +basop-ref-compat-encoder-stv: extends: - .basop-ci-branch-compat-template variables: @@ -515,7 +515,7 @@ check-compatibility-with-basop-reference-branch-encoder-stv: script: - exit 0 -check-compatibility-with-basop-reference-branch-encoder-ltv: +basop-ref-compat-encoder-ltv: extends: - .basop-ci-branch-compat-template variables: @@ -523,7 +523,7 @@ check-compatibility-with-basop-reference-branch-encoder-ltv: script: - exit 0 -check-compatibility-with-basop-reference-branch-encoder-dmx-comparison: +basop-ref-compat-encoder-dmx-comp: extends: - .basop-ci-branch-compat-template variables: -- GitLab From 4751b1ee7c1bd6aa7bda9e8ed32f7a1968a9743d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 10 Feb 2025 14:47:15 +0100 Subject: [PATCH 21/46] correct pytest argument --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3f0283ba1e..36b589383f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -529,7 +529,7 @@ basop-ref-compat-encoder-dmx-comp: variables: PARAM_FILE: scripts/config/self_test_basop_encoder.prm # USING PYTEST_ADDOPTS env var did not work for some reason when testing locally - maybe because this is a custom option - PYTEST_ARGS: "--compare_dmx_enc" + PYTEST_ARGS: "--compare_enc_dmx" script: - exit 0 -- GitLab From f4cae5563189a87b604cd09d93551484e86fe55d Mon Sep 17 00:00:00 2001 From: rtyag Date: Tue, 18 Feb 2025 18:44:08 +1100 Subject: [PATCH 22/46] instrumentation fix for split rendering --- scripts/prepare_instrumentation.sh | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/scripts/prepare_instrumentation.sh b/scripts/prepare_instrumentation.sh index 4ed7098442..35bbfe4420 100755 --- a/scripts/prepare_instrumentation.sh +++ b/scripts/prepare_instrumentation.sh @@ -160,6 +160,12 @@ if [ $? -ne 0 ]; then exit -1 fi +# Floating point code has lib_lc3plus/fft folder while BASOP code does not +fftdir="" +if [ -d $targetdir/lib_lc3plus/fft ]; then + fftdir="$targetdir/lib_lc3plus/fft/*.c" +fi + # strip switches, to remove the macros (turn on extended globing to allow !(pattern*) matching) shopt -s extglob if coan_exists; then @@ -171,7 +177,9 @@ if coan_exists; then if [ "$PROJECT" = "FLOAT" ]; then coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_{com,dec,enc,isar,rend,util,debug}/!(wmc_auto*).[hc] coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_lc3plus/!(wmc_auto*).[hc] - coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_lc3plus/fft/!(wmc_auto*).[hc] + if [ -n "$fftdir" ];then + coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_lc3plus/fft/!(wmc_auto*).[hc] + fi else # same as first call from if, but without "isar" and "debug" to avoid coan warning coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_{com,dec,enc,rend,util}/!(wmc_auto*).[hc] @@ -185,18 +193,20 @@ shopt -u extglob find $targetdir -name "*.[ch]" -exec sed -i.bak -e "s/\(0x[0-9a-fA-F]*\)UL/\(\(unsigned long\)\1\)/" \{\} \; # run wmc_tool -"tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/encoder.c" "$targetdir/lib_enc/*.c" "$targetdir/lib_com/*.c" >> /dev/null -"tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/decoder.c" "$targetdir/lib_dec/*.c" "$targetdir/lib_rend/*.c" >> /dev/null +"tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/encoder.c" "$targetdir/lib_enc/*.c" "$targetdir/lib_com/*.c" >> output.txt 2>&1 +"tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/decoder.c" "$targetdir/lib_dec/*.c" "$targetdir/lib_rend/*.c" >> output.txt 2>&1 # ISAR post-renderer and lc3plus sources only need to be instrumented in float code if [ "$PROJECT" = "FLOAT" ]; then for bak_file in $targetdir/lib_rend/*.bak; do mv "$bak_file" "${bak_file%.*}"; done # restore fresh .c files to avoid time-consuming des-instrumentation of files by the WMC tool - "tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/renderer.c" "$targetdir/lib_rend/*.c" "$targetdir/lib_lc3plus/*.c" "$targetdir/lib_lc3plus/fft/*.c" >> /dev/null + "tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/renderer.c" "$targetdir/lib_rend/*.c" "$targetdir/lib_lc3plus/*.c" "$fftdir" >> output.txt 2>&1 for bak_file in $targetdir/lib_lc3plus/*.bak; do mv "$bak_file" "${bak_file%.*}"; done # restore fresh .c files to avoid time-consuming des-instrumentation of files by the WMC tool - for bak_file in $targetdir/lib_lc3plus/fft/*.bak; do mv "$bak_file" "${bak_file%.*}"; done # restore fresh .c files to avoid time-consuming des-instrumentation of files by the WMC tool - "tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/isar_post_rend.c" "$targetdir/lib_isar/*.c" "$targetdir/lib_lc3plus/*.c" "$targetdir/lib_lc3plus/fft/*.c" >> /dev/null + if [ -n "$fftdir" ];then + for bak_file in $targetdir/lib_lc3plus/fft/*.bak; do mv "$bak_file" "${bak_file%.*}"; done # restore fresh .c files to avoid time-consuming des-instrumentation of files by the WMC tool + fi + "tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/isar_post_rend.c" "$targetdir/lib_isar/*.c" "$targetdir/lib_lc3plus/*.c" "$fftdir" >> output.txt 2>&1 else for bak_file in $targetdir/lib_rend/*.bak; do mv "$bak_file" "${bak_file%.*}"; done # restore fresh .c files to avoid time-consuming des-instrumentation of files by the WMC tool - "tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/renderer.c" "$targetdir/lib_rend/*.c" >> /dev/null + "tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/renderer.c" "$targetdir/lib_rend/*.c" >> output.txt 2>&1 fi # automatically enable #define WMOPS in options.h -- GitLab From d964a0a0c68b1afbfaec3d23fad4d85f71745053 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Feb 2025 10:48:04 +0100 Subject: [PATCH 23/46] add script for creating ignorelist for ubsan ignore all non-ivas_* files in ubsan for now --- scripts/basop_create_ignorelist_for_ubsan.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 scripts/basop_create_ignorelist_for_ubsan.py diff --git a/scripts/basop_create_ignorelist_for_ubsan.py b/scripts/basop_create_ignorelist_for_ubsan.py new file mode 100644 index 0000000000..7369fc6c9f --- /dev/null +++ b/scripts/basop_create_ignorelist_for_ubsan.py @@ -0,0 +1,9 @@ +import pathlib + +all_c_files = pathlib.Path(".").glob("lib_*/*.c") +ivas_c_files = pathlib.Path(".").glob("lib_*/ivas*.c") +non_ivas_c_files = sorted(set(all_c_files) - set(ivas_c_files)) + +with open("ubsan_ignorelist.txt", "w") as f: + for cfile in non_ivas_c_files: + print(f"src:{cfile}", file=f) -- GitLab From 03cb598a2de0074840ba10ef2069d3267fd6247f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Feb 2025 14:23:45 +0100 Subject: [PATCH 24/46] also add header files to ignorelist e.g. enh40.h contains function code --- scripts/basop_create_ignorelist_for_ubsan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/basop_create_ignorelist_for_ubsan.py b/scripts/basop_create_ignorelist_for_ubsan.py index 7369fc6c9f..e6dfdfa075 100644 --- a/scripts/basop_create_ignorelist_for_ubsan.py +++ b/scripts/basop_create_ignorelist_for_ubsan.py @@ -1,7 +1,7 @@ import pathlib -all_c_files = pathlib.Path(".").glob("lib_*/*.c") -ivas_c_files = pathlib.Path(".").glob("lib_*/ivas*.c") +all_c_files = pathlib.Path(".").glob("lib_*/*.[ch]") +ivas_c_files = pathlib.Path(".").glob("lib_*/ivas*.[ch]") non_ivas_c_files = sorted(set(all_c_files) - set(ivas_c_files)) with open("ubsan_ignorelist.txt", "w") as f: -- GitLab From 183c425143626d2602c27150ac4fb20926b4fe10 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 24 Feb 2025 12:41:54 +0100 Subject: [PATCH 25/46] add script for getting the ref branch of a basop port branch --- ci/get_float_ref_branch_name.py | 59 +++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 ci/get_float_ref_branch_name.py diff --git a/ci/get_float_ref_branch_name.py b/ci/get_float_ref_branch_name.py new file mode 100644 index 0000000000..b8da8979e7 --- /dev/null +++ b/ci/get_float_ref_branch_name.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +""" +(C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +import re +import sys +import argparse + +BASOP_PATTERN = r"\d+[_-]basop[_-].*" + +MSG_DOES_NOT_MATCH = """Your branch name {branchname} does not match the template '_basop_', e.g. '123_basop_fix_this_one_bug-2'. +Sticking to this branch is needed for the testing system to match this branch with its float-reference counterpart. +Please rename your branch. You can easily do this by creating a new branch from this branch: + - git checkout -b +You then also need to create a new merge request and update the links in your issue. +See here for details on the porting work process: https://forge.3gpp.org/rep/sa4/audio/ivas-basop/-/wikis/Porting-MRs-from-floating-point-codec#workflow-for-porting-a-merge-request-from-floating-point-codec-to-basop-codec +""" + +parser = argparse.ArgumentParser() +parser.add_argument("branchname") +args = parser.parse_args() +branchname = args.branchname + +re_match = re.match(BASOP_PATTERN, branchname) +if re_match is None: + print(MSG_DOES_NOT_MATCH.format(branchname=branchname)) + sys.exit(1) + +# the float ref branch is just the same name, but with basop -> ref replacement +# replace only the first occurrence, as "basop" mit be present in the later description +# if the format is correct, then before "_basop", only numbers can occur +print(branchname.replace("basop", "ref", 1)) -- GitLab From 7247d074c01462138b878b1339b8ccdf39902a7d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 25 Feb 2025 11:07:07 +0100 Subject: [PATCH 26/46] extend script + turn into bash script --- ci/get_float_ref_branch_name.py | 59 ------------------------------ ci/get_float_ref_branch_name.sh | 65 +++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 59 deletions(-) delete mode 100644 ci/get_float_ref_branch_name.py create mode 100644 ci/get_float_ref_branch_name.sh diff --git a/ci/get_float_ref_branch_name.py b/ci/get_float_ref_branch_name.py deleted file mode 100644 index b8da8979e7..0000000000 --- a/ci/get_float_ref_branch_name.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 -""" -(C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, -Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., -Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, -Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other -contributors to this repository. All Rights Reserved. - -This software is protected by copyright law and by international treaties. -The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, -Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., -Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, -Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other -contributors to this repository retain full ownership rights in their respective contributions in -the software. This notice grants no license of any kind, including but not limited to patent -license, nor is any license granted by implication, estoppel or otherwise. - -Contributors are required to enter into the IVAS codec Public Collaboration agreement before making -contributions. - -This software is provided "AS IS", without any express or implied warranties. The software is in the -development stage. It is intended exclusively for experts who have experience with such software and -solely for the purpose of inspection. All implied warranties of non-infringement, merchantability -and fitness for a particular purpose are hereby disclaimed and excluded. - -Any dispute, controversy or claim arising under or in relation to providing this software shall be -submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in -accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and -the United Nations Convention on Contracts on the International Sales of Goods. -""" - -import re -import sys -import argparse - -BASOP_PATTERN = r"\d+[_-]basop[_-].*" - -MSG_DOES_NOT_MATCH = """Your branch name {branchname} does not match the template '_basop_', e.g. '123_basop_fix_this_one_bug-2'. -Sticking to this branch is needed for the testing system to match this branch with its float-reference counterpart. -Please rename your branch. You can easily do this by creating a new branch from this branch: - - git checkout -b -You then also need to create a new merge request and update the links in your issue. -See here for details on the porting work process: https://forge.3gpp.org/rep/sa4/audio/ivas-basop/-/wikis/Porting-MRs-from-floating-point-codec#workflow-for-porting-a-merge-request-from-floating-point-codec-to-basop-codec -""" - -parser = argparse.ArgumentParser() -parser.add_argument("branchname") -args = parser.parse_args() -branchname = args.branchname - -re_match = re.match(BASOP_PATTERN, branchname) -if re_match is None: - print(MSG_DOES_NOT_MATCH.format(branchname=branchname)) - sys.exit(1) - -# the float ref branch is just the same name, but with basop -> ref replacement -# replace only the first occurrence, as "basop" mit be present in the later description -# if the format is correct, then before "_basop", only numbers can occur -print(branchname.replace("basop", "ref", 1)) diff --git a/ci/get_float_ref_branch_name.sh b/ci/get_float_ref_branch_name.sh new file mode 100644 index 0000000000..0d791b6f5c --- /dev/null +++ b/ci/get_float_ref_branch_name.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +# contributors to this repository. All Rights Reserved. +# +# This software is protected by copyright law and by international treaties. +# The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +# contributors to this repository retain full ownership rights in their respective contributions in +# the software. This notice grants no license of any kind, including but not limited to patent +# license, nor is any license granted by implication, estoppel or otherwise. +# +# Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +# contributions. +# +# This software is provided "AS IS", without any express or implied warranties. The software is in the +# development stage. It is intended exclusively for experts who have experience with such software and +# solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +# and fitness for a particular purpose are hereby disclaimed and excluded. +# +# Any dispute, controversy or claim arising under or in relation to providing this software shall be +# submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +# accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +# the United Nations Convention on Contracts on the International Sales of Goods. + +BASOP_PATTERN="^[0-9]+[_-]basop[_-].*" +MSG_DOES_NOT_MATCH="Your branch name %s does not match the template '_basop_', e.g. '123_basop_fix_this_one_bug-2'. +Sticking to this branch is needed for the testing system to match this branch with its float-reference counterpart. +Please rename your branch. You can easily do this by creating a new branch from this branch: + - git checkout -b +You then also need to create a new merge request and update the links in your issue. +See here for details on the porting work process: https://forge.3gpp.org/rep/sa4/audio/ivas-basop/-/wikis/Porting-MRs-from-floating-point-codec#workflow-for-porting-a-merge-request-from-floating-point-codec-to-basop-codec +" + +if [ $# -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +branchname="$1" + +if ! [[ "$branchname" =~ $BASOP_PATTERN ]]; then + printf "$MSG_DOES_NOT_MATCH" "$branchname" + exit 1 +fi + +float_ref_branchname="${branchname/basop/ref}" +git_result=$(git branch --list "$float_ref_branchname") + +# If the branch does not exist, default to "float-pc" +if [[ -z "$git_result" ]]; then + branchname="float-pc" +else + # The float ref branch is just the same name, but with basop -> ref replacement + # Replace only the first occurrence, as "basop" may be present in the later description + # If the format is correct, then before "_basop", only numbers can occur + branchname="${float_ref_branchname}" +fi + +echo "$branchname" -- GitLab From d89d577d131ba88c8bb0c2adc88a9fac643f5cfb Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 25 Feb 2025 12:57:57 +0100 Subject: [PATCH 27/46] make script executable --- ci/get_float_ref_branch_name.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ci/get_float_ref_branch_name.sh diff --git a/ci/get_float_ref_branch_name.sh b/ci/get_float_ref_branch_name.sh old mode 100644 new mode 100755 -- GitLab From b99a2e5f599bb210acc4f00766e6ded59be87721 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Feb 2025 10:23:58 +0100 Subject: [PATCH 28/46] report regressions and improvements in separate files --- .../basop_check_for_changes_in_testcases.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index b47f0d6cc2..1ac3dfca18 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -37,7 +37,6 @@ import os import pathlib import re import xml.etree.ElementTree as ET -from typing import Tuple # set positive threshold for "lower is better" metrics, negative for "higher is better" @@ -48,8 +47,10 @@ COLS_2_THRESHOLDS = { "MIN_ODG": float(os.environ.get("CI_REGRESSION_THRESH_ODG", -0.05)), } -OUTFILE_CRASHES = "changes_crashes.csv" -OUTFILE_SCORES = "changes_{}.csv" +OUTFILE_CRASHES_FIXED = "improvements_crashes.csv" +OUTFILE_CRASHES_ADDED = "regressions_crashes.csv" +OUTFILE_SCORES_IMPROVEMENTS = "improvements_{}.csv" +OUTFILE_SCORES_REGRESSION = "regressions_{}.csv" PATTERN_ENC = r"... encoder command:\s*(.*)\s*... encoder stdout:" PATTERN_DEC = r"... decoder command:\s*(.*)\s*... decoder stdout:" @@ -102,7 +103,7 @@ def main(args): df_crashes_introduced = pd.merge( df_crashes_introduced, cmdlines_crashes_introduced, on="testcase" ) - df_crashes_introduced.to_csv(OUTFILE_CRASHES, sep=";") + df_crashes_introduced.to_csv(OUTFILE_CRASHES_ADDED, sep=";") if args.show_improvements and sum(mask_crash_fixed) > 0: df_crashes_fixed = df_merged[mask_crash_fixed][display_cols].reset_index( @@ -119,7 +120,7 @@ def main(args): df_crashes_fixed = pd.merge( df_crashes_fixed, cmdlines_crashes_fixed, on="testcase" ) - df_crashes_fixed.to_csv(OUTFILE_CRASHES, mode="a", sep=";") + df_crashes_fixed.to_csv(OUTFILE_CRASHES_FIXED, mode="a", sep=";") # remove columns with ERRORs in any of the csv files before comparing the numerical columns mask_no_errors = (df_merged[col_curr] != "ERROR") & (df_merged[col_prev] != "ERROR") @@ -140,7 +141,7 @@ def main(args): mask_better = (df_merged[col_diff] * fac) < -thresh display_cols = ["testcase", col_curr, col_prev, col_diff] - outfile = OUTFILE_SCORES.format(col.replace(" ", "_")) + outfile_regressions = OUTFILE_SCORES_REGRESSION.format(col.replace(" ", "_")) df_worse = df_merged[mask_worse][display_cols].reset_index(drop=True) if sum(mask_worse) > 0: regressions_found = True @@ -155,7 +156,7 @@ def main(args): df_worse["testcase"], xml_report, args.inject_cwd ) df_worse = pd.merge(df_worse, cmdlines_worse, on="testcase") - df_worse.to_csv(outfile, sep=";") + df_worse.to_csv(outfile_regressions, sep=";") if args.show_improvements and sum(mask_better) > 0: df_better = df_merged[mask_better][display_cols].reset_index(drop=True) @@ -170,7 +171,10 @@ def main(args): df_better["testcase"], xml_report, args.inject_cwd ) df_better = pd.merge(df_better, cmdlines_better, on="testcase") - df_better.to_csv(outfile, mode="a", sep=";") + outfile_improvements = OUTFILE_SCORES_IMPROVEMENTS.format( + col.replace(" ", "_") + ) + df_better.to_csv(outfile_improvements, mode="a", sep=";") return int(regressions_found) -- GitLab From de027e8b9a58935378d96c61edcbafd9cf495aea Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Feb 2025 10:37:28 +0100 Subject: [PATCH 29/46] remove now obsolete append mode --- scripts/basop_check_for_changes_in_testcases.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index 1ac3dfca18..315afd0a88 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -120,7 +120,7 @@ def main(args): df_crashes_fixed = pd.merge( df_crashes_fixed, cmdlines_crashes_fixed, on="testcase" ) - df_crashes_fixed.to_csv(OUTFILE_CRASHES_FIXED, mode="a", sep=";") + df_crashes_fixed.to_csv(OUTFILE_CRASHES_FIXED, sep=";") # remove columns with ERRORs in any of the csv files before comparing the numerical columns mask_no_errors = (df_merged[col_curr] != "ERROR") & (df_merged[col_prev] != "ERROR") @@ -174,7 +174,7 @@ def main(args): outfile_improvements = OUTFILE_SCORES_IMPROVEMENTS.format( col.replace(" ", "_") ) - df_better.to_csv(outfile_improvements, mode="a", sep=";") + df_better.to_csv(outfile_improvements, sep=";") return int(regressions_found) -- GitLab From 3220d72fe9b814a97dab30ff48ae6800d0c467e7 Mon Sep 17 00:00:00 2001 From: rtyag Date: Mon, 3 Mar 2025 15:05:24 +1100 Subject: [PATCH 30/46] add MLD, ODG and SSNR to split rendering CI --- tests/split_rendering/test_split_rendering.py | 246 +++++++++++++++++- tests/split_rendering/utils.py | 79 +++++- 2 files changed, 305 insertions(+), 20 deletions(-) diff --git a/tests/split_rendering/test_split_rendering.py b/tests/split_rendering/test_split_rendering.py index 24360208ab..7403669251 100644 --- a/tests/split_rendering/test_split_rendering.py +++ b/tests/split_rendering/test_split_rendering.py @@ -44,12 +44,21 @@ from tests.split_rendering.utils import * @pytest.mark.parametrize("bitrate", IVAS_BITRATES_AMBI) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI_SPLIT_REND) def test_ambisonics_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, test_info, in_fmt, bitrate, render_config, trajectory ): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -57,6 +66,11 @@ def test_ambisonics_full_chain_split( binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -64,16 +78,31 @@ def test_ambisonics_full_chain_split( @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_AMBI) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI_SPLIT_REND) -def test_ambisonics_external_split(test_info, in_fmt, render_config, trajectory): +def test_ambisonics_external_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + test_info, in_fmt, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -86,12 +115,21 @@ def test_ambisonics_external_split(test_info, in_fmt, render_config, trajectory) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_MC) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC_SPLIT_REND) def test_multichannel_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, test_info, in_fmt, bitrate, render_config, trajectory ): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -99,6 +137,11 @@ def test_multichannel_full_chain_split( binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -106,16 +149,31 @@ def test_multichannel_full_chain_split( @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MC) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC_SPLIT_REND) -def test_multichannel_external_split(test_info, in_fmt, render_config, trajectory): +def test_multichannel_external_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -127,11 +185,21 @@ def test_multichannel_external_split(test_info, in_fmt, render_config, trajector @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_ISM) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_ISM) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM_SPLIT_REND) -def test_ism_full_chain_split(test_info, in_fmt, bitrate, render_config, trajectory): +def test_ism_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, bitrate, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -139,6 +207,11 @@ def test_ism_full_chain_split(test_info, in_fmt, bitrate, render_config, traject binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -146,16 +219,31 @@ def test_ism_full_chain_split(test_info, in_fmt, bitrate, render_config, traject @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_ISM) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM_SPLIT_REND) -def test_ism_external_split(test_info, in_fmt, render_config, trajectory): +def test_ism_external_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -167,11 +255,21 @@ def test_ism_external_split(test_info, in_fmt, render_config, trajectory): @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MASA) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_MASA) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA_SPLIT_REND) -def test_masa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajectory): +def test_masa_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, bitrate, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -179,6 +277,11 @@ def test_masa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -186,16 +289,31 @@ def test_masa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MASA) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA_SPLIT_REND) -def test_masa_external_split(test_info, in_fmt, render_config, trajectory): +def test_masa_external_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -207,11 +325,21 @@ def test_masa_external_split(test_info, in_fmt, render_config, trajectory): @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_OMASA) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_OMASA) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_OMASA_SPLIT_REND) -def test_omasa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajectory): +def test_omasa_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, bitrate, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -219,6 +347,11 @@ def test_omasa_full_chain_split(test_info, in_fmt, bitrate, render_config, traje binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -230,11 +363,21 @@ def test_omasa_full_chain_split(test_info, in_fmt, bitrate, render_config, traje @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_OSBA) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_OSBA) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_OSBA_SPLIT_REND) -def test_osba_full_chain_split(test_info, in_fmt, bitrate, render_config, trajectory): +def test_osba_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, bitrate, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -242,6 +385,11 @@ def test_osba_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -253,17 +401,32 @@ def test_osba_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_PLC) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI_SPLIT_REND[-1:]) -def test_post_rend_plc(test_info, in_fmt, render_config, trajectory, error_pattern): +def test_post_rend_plc( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config, trajectory, error_pattern): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, plc_error_pattern=ERROR_PATTERNS_DIR.joinpath(f"{error_pattern}.ep"), + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -281,12 +444,22 @@ full_chain_split_pcm_params = [ @pytest.mark.parametrize("in_fmt,bitrate,render_config", full_chain_split_pcm_params) -def test_full_chain_split_pcm(test_info, in_fmt, bitrate, render_config): +def test_full_chain_split_pcm( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, bitrate, render_config): trajectory = SPLIT_REND_HR_TRAJECTORIES_TO_TEST[0] post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -295,6 +468,11 @@ def test_full_chain_split_pcm(test_info, in_fmt, bitrate, render_config): pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, renderer_fmt="BINAURAL_SPLIT_PCM", + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -307,18 +485,33 @@ external_split_pcm_params = [ @pytest.mark.parametrize("in_fmt,render_config", external_split_pcm_params) -def test_external_split_pcm(test_info, in_fmt, render_config): +def test_external_split_pcm( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config): trajectory = SPLIT_REND_HR_TRAJECTORIES_TO_TEST[0] post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, renderer_fmt="BINAURAL_SPLIT_PCM", + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -327,11 +520,21 @@ def test_external_split_pcm(test_info, in_fmt, render_config): @pytest.mark.parametrize("in_fmt", ["5_1"]) @pytest.mark.parametrize("pre_rend_fr", SPLIT_RENDERER_PRE_FRAMINGS) @pytest.mark.parametrize("post_rend_fr", SPLIT_RENDERER_POST_FRAMINGS) -def test_framing_combinations_external_split(test_info, in_fmt, render_config, trajectory, post_rend_fr, pre_rend_fr): +def test_framing_combinations_external_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config, trajectory, post_rend_fr, pre_rend_fr): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_FRAMING_CFG_DIR.joinpath(f"{render_config}.txt"), @@ -339,6 +542,11 @@ def test_framing_combinations_external_split(test_info, in_fmt, render_config, t post_trajectory=post_trajectory, post_rend_fr=post_rend_fr, pre_rend_fr=pre_rend_fr, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @@ -347,12 +555,21 @@ def test_framing_combinations_external_split(test_info, in_fmt, render_config, t @pytest.mark.parametrize("pre_rend_fr", SPLIT_RENDERER_PRE_FRAMINGS) @pytest.mark.parametrize("post_rend_fr", SPLIT_RENDERER_POST_FRAMINGS) def test_framing_combinations_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, test_info, in_fmt, render_config, trajectory, post_rend_fr, pre_rend_fr ): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_FRAMING_CFG_DIR.joinpath(f"{render_config}.txt"), @@ -362,4 +579,9 @@ def test_framing_combinations_full_chain_split( binary_suffix=EXE_SUFFIX, post_rend_fr=post_rend_fr, pre_rend_fr=pre_rend_fr, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) diff --git a/tests/split_rendering/utils.py b/tests/split_rendering/utils.py index 504929b29a..b8aa6737ea 100644 --- a/tests/split_rendering/utils.py +++ b/tests/split_rendering/utils.py @@ -41,6 +41,8 @@ import pytest from tests.renderer.utils import check_BE, run_ivas_isar_enc_cmd, run_ivas_isar_dec_cmd, run_isar_post_rend_cmd, run_isar_ext_rend_cmd from tests.split_rendering.constants import * +from ..cmp_pcm import cmp_pcm +from ..conftest import parse_properties sys.path.append(SCRIPTS_DIR) from pyaudio3dtools.audiofile import readfile, writefile @@ -151,6 +153,8 @@ def truncate_signal( def run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt: str, bitrate: str, @@ -161,6 +165,11 @@ def run_full_chain_split_rendering( binary_suffix: str = "", post_rend_fr: str = "20", pre_rend_fr: str = "20", + get_mld=False, + mld_lim=0, + get_ssnr=False, + get_odg=False, + get_odg_bin=False, ) -> str: """ Runs the full split rendering chain consisting of @@ -274,15 +283,41 @@ def run_full_chain_split_rendering( cut, cut_fs = readfile(out_file) - [diff_found, snr, gain_b, max_diff] = check_BE(test_info, ref, ref_fs, cut, cut_fs) - if diff_found : - pytest.fail( - f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" + if get_mld == False: + [diff_found, snr, gain_b, max_diff] = check_BE(test_info, ref, ref_fs, cut, cut_fs) + if diff_found : + pytest.fail( + f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" + ) + else: + # see constants.py + output_differs, reason = cmp_pcm( + out_file, + out_file_ref, + "BINAURAL", + ref_fs, + get_mld=get_mld, + mld_lim=mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) + + props = parse_properties(reason, output_differs, props_to_record) + for k, v in props.items(): + record_property(k, v) + + if output_differs: + pytest.fail(f"Output differs: ({reason})") + + + return out_file def run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt: str, render_config: Path, @@ -294,6 +329,11 @@ def run_external_split_rendering( is_comparetest: bool = False, post_rend_fr: str = "20", pre_rend_fr: str = "20", + get_mld=False, + mld_lim=0, + get_ssnr=False, + get_odg=False, + get_odg_bin=False, ) -> Tuple[np.ndarray, int]: """ Runs the exeternal split rendering chain consisting of @@ -393,9 +433,32 @@ def run_external_split_rendering( cut, cut_fs = readfile(out_file) - [diff_found, snr, gain_b, max_diff] = check_BE(test_info, ref, ref_fs, cut, cut_fs) - if diff_found : - pytest.fail( - f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" + if get_mld == False: + [diff_found, snr, gain_b, max_diff] = check_BE(test_info, ref, ref_fs, cut, cut_fs) + if diff_found : + pytest.fail( + f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" + ) + else: + # see constants.py + output_differs, reason = cmp_pcm( + out_file, + out_file_ref, + "BINAURAL", + ref_fs, + get_mld=get_mld, + mld_lim=mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) + + props = parse_properties(reason, output_differs, props_to_record) + for k, v in props.items(): + record_property(k, v) + + if output_differs: + pytest.fail(f"Output differs: ({reason})") + + return out_file -- GitLab From 55754c39f179f7feb9bb03367d9af01d30303cf5 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 12 Mar 2025 08:44:10 +0100 Subject: [PATCH 31/46] remove basop files from usan ignorelist --- scripts/basop_create_ignorelist_for_ubsan.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/basop_create_ignorelist_for_ubsan.py b/scripts/basop_create_ignorelist_for_ubsan.py index e6dfdfa075..d13c0d5f5b 100644 --- a/scripts/basop_create_ignorelist_for_ubsan.py +++ b/scripts/basop_create_ignorelist_for_ubsan.py @@ -1,9 +1,14 @@ import pathlib -all_c_files = pathlib.Path(".").glob("lib_*/*.[ch]") -ivas_c_files = pathlib.Path(".").glob("lib_*/ivas*.[ch]") -non_ivas_c_files = sorted(set(all_c_files) - set(ivas_c_files)) +here = pathlib.Path(".") + +all_files = here.glob("lib_*/*.[ch]") +ivas_files = here.glob("lib_*/ivas*.[ch]") +non_ivas_files = sorted(set(all_files) - set(ivas_files)) +basop_files = here.glob("lib_*/basop*.[ch]") + +ignorefiles = sorted(set(non_ivas_files) - set(basop_files)) with open("ubsan_ignorelist.txt", "w") as f: - for cfile in non_ivas_c_files: + for cfile in ignorefiles: print(f"src:{cfile}", file=f) -- GitLab From f29a79fc86a3135cd8d1fb389f81c439cb92eb50 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Mar 2025 14:11:33 +0100 Subject: [PATCH 32/46] make git check for branch on remote as well in CI container, sometimes the reference branch was not locally available and fetching did not bring it in scope... --- ci/get_float_ref_branch_name.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/get_float_ref_branch_name.sh b/ci/get_float_ref_branch_name.sh index 0d791b6f5c..6c183bfb48 100755 --- a/ci/get_float_ref_branch_name.sh +++ b/ci/get_float_ref_branch_name.sh @@ -50,7 +50,7 @@ if ! [[ "$branchname" =~ $BASOP_PATTERN ]]; then fi float_ref_branchname="${branchname/basop/ref}" -git_result=$(git branch --list "$float_ref_branchname") +git_result=$(git branch -av --list "$float_ref_branchname") # If the branch does not exist, default to "float-pc" if [[ -z "$git_result" ]]; then -- GitLab From 2ded29b229cb4db42b1ca2cfd33178541867a5b8 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Mar 2025 14:53:17 +0100 Subject: [PATCH 33/46] change check for ref branch existing --- ci/get_float_ref_branch_name.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ci/get_float_ref_branch_name.sh b/ci/get_float_ref_branch_name.sh index 6c183bfb48..fe160da023 100755 --- a/ci/get_float_ref_branch_name.sh +++ b/ci/get_float_ref_branch_name.sh @@ -49,17 +49,17 @@ if ! [[ "$branchname" =~ $BASOP_PATTERN ]]; then exit 1 fi +# The float ref branch is just the same name, but with basop -> ref replacement +# Replace only the first occurrence, as "basop" may be present in the later description +# If the format is correct, then before "_basop", only numbers can occur float_ref_branchname="${branchname/basop/ref}" -git_result=$(git branch -av --list "$float_ref_branchname") +git_result=$(git branch -av) # If the branch does not exist, default to "float-pc" -if [[ -z "$git_result" ]]; then - branchname="float-pc" -else - # The float ref branch is just the same name, but with basop -> ref replacement - # Replace only the first occurrence, as "basop" may be present in the later description - # If the format is correct, then before "_basop", only numbers can occur +if [[ "$git_result" =~ "$float_ref_branchname" ]]; then branchname="${float_ref_branchname}" +else + branchname="float-pc" fi echo "$branchname" -- GitLab From d74b128392f978031965dccef889a40f89df8e34 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 19 Mar 2025 09:38:55 +0100 Subject: [PATCH 34/46] add extra set of testcases to remove for main/float-pc branches --- ci/remove_unsupported_testcases.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/ci/remove_unsupported_testcases.py b/ci/remove_unsupported_testcases.py index a4a0496809..e5e184fa83 100644 --- a/ci/remove_unsupported_testcases.py +++ b/ci/remove_unsupported_testcases.py @@ -32,7 +32,7 @@ from pathlib import Path import argparse # Enter tag of testcases to remove here WITHOUT the leading // -TESTCASES = [ +TESTCASES_MAIN = [ "OMASA 2Dir2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out", "OSBA planar FOA 2ISM at 512 kbps, 48 kHz in, 48 kHz out, BINAURAL out", "4 ISM with extended metadata at 128 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out, rendconf dir w id", @@ -86,6 +86,20 @@ TESTCASES = [ "4 ISM with metadata bitrate switching from 32 kbps to 48 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_IR out, object editing", "4 ISM with metadata bitrate switching from 48 kbps to 32 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, JBM Prof 5", ] +TESTCASES_MAIN_PC = [ + "SBA at 128 kbps, 32kHZ in, 32kHz out, BINAURAL_ROOM_REVERB out, Config renderer, HR", + "SBA at 128 kbps, 32kHZ in, 16kHz out, BINAURAL_ROOM_REVERB out (Model from file), HR", + "Planar SBA at 128 kbps, 48kHZ in, 32kHz out, BINAURAL_ROOM_REVERB out (Model from file), Config renderer, HR", + "Multi-channel 7_1_4 at 160 kbps, 48kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out default configuration", + "Multi-channel 5_1 at 80 kbps, 48kHz in, 32kHz out, BINAURAL_ROOM_REVERB out Config renderer, HR", + "OSBA planar 2OA 4ISM at 512 kbps, 48 kHz in, 48 kHz out, BINAURAL ROOM REVERB (Model from file) out", + "SBA 3OA 4ISM at 96 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out custom configuration", + "Multi-channel 7_1_4 at 160 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out Config recreation, HR", + "Multi-channel 5_1_2 at 64 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out Config renderer, HR", + "OSBA 3OA 4ISM at 512 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out", + "OSBA 3ISM 3OA at bitrate switching 13.2 to 512 kbps, 48kHz in, 32kHz out, BINAURAL ROOM REVERB out", + "Multi-channel 5_1 bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 16kHz out, BINAURAL_ROOM_REVERB out", +] def remove_testcases(cfg: Path, testcases: list): @@ -113,7 +127,12 @@ def remove_testcases(cfg: Path, testcases: list): if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("cfg_files", nargs="+", type=Path) + parser.add_argument("--use-main-pc-set", action="store_true") args = parser.parse_args() + testcases = TESTCASES_MAIN + if args.use_main_pc_set: + testcases.extend(TESTCASES_MAIN_PC) + for f in args.cfg_files: - remove_testcases(f, TESTCASES) + remove_testcases(f, testcases) -- GitLab From 814385b175eba0f4e5aa9cc84bc1d6300cf97d97 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 2 Apr 2025 11:17:10 +0200 Subject: [PATCH 35/46] some adjustments to basop compat CI --- .gitlab-ci.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1721018369..b1d952ddad 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -134,8 +134,11 @@ stages: - sed -i.bak -e "s/\/\/\ *\(#define\ *DEBUGGING\ *\)/\1/g" lib_com/options.h .get-basop-float-reference: &get-basop-float-reference - - git clone -b $BASOP_REFERENCE_BRANCH https://forge.3gpp.org/rep/sa4/audio/ivas-basop.git --single-branch - - make -j -C ivas-basop + - git clone -b $BASOP_REFERENCE_BRANCH https://forge.3gpp.org/rep/sa4/audio/ivas-basop.git --single-branch --depth 1 ivas-basop + - cd ivas-basop + - git status + - make -j + - cd - - cp ivas-basop/IVAS_cod ./ - cp ivas-basop/IVAS_dec ./ @@ -463,7 +466,7 @@ branch-is-up-to-date-with-main-post: - .test-job-linux - .rules-merge-request-to-basop-ci-branch tags: - - ivas-linux-fast + - ivas-linux before_script: - python3 ci/remove_unsupported_testcases.py $PARAM_FILE - python3 tests/create_short_testvectors.py @@ -502,6 +505,8 @@ basop-ref-compat-stv: basop-ref-compat-ltv: extends: - .basop-ci-branch-compat-template + tags: + - ivas-linux-fast variables: PARAM_FILE: scripts/config/self_test_ltv.prm script: @@ -518,6 +523,8 @@ basop-ref-compat-encoder-stv: basop-ref-compat-encoder-ltv: extends: - .basop-ci-branch-compat-template + tags: + - ivas-linux-fast variables: PARAM_FILE: scripts/config/self_test_ltv_basop_encoder.prm script: -- GitLab From 772458c220d25506fecc839b7d5cf7d74dc80c33 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 2 Apr 2025 11:44:00 +0200 Subject: [PATCH 36/46] add timeouts for basop-compat tests + new unsupported case --- .gitlab-ci.yml | 7 ++++++- ci/remove_unsupported_testcases.py | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b1d952ddad..b6e4433136 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -477,7 +477,7 @@ branch-is-up-to-date-with-main-post: - *get-basop-float-reference - - python3 -m pytest tests/codec_be_on_mr_nonselection $PYTEST_ARGS --param_file $PARAM_FILE --update_ref 1 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml || true + - python3 -m pytest tests/codec_be_on_mr_nonselection $PYTEST_ARGS --param_file $PARAM_FILE --update_ref 1 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$TESTCASE_TIMEOUT || true - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 1 ]; then echo "Run errors encountered with $PARAM_FILE !"; exit $EXIT_CODE_FAIL; fi @@ -499,6 +499,7 @@ basop-ref-compat-stv: - .basop-ci-branch-compat-template variables: PARAM_FILE: scripts/config/self_test.prm + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS script: - exit 0 @@ -509,6 +510,7 @@ basop-ref-compat-ltv: - ivas-linux-fast variables: PARAM_FILE: scripts/config/self_test_ltv.prm + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS script: - exit 0 @@ -517,6 +519,7 @@ basop-ref-compat-encoder-stv: - .basop-ci-branch-compat-template variables: PARAM_FILE: scripts/config/self_test_basop_encoder.prm + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS script: - exit 0 @@ -527,6 +530,7 @@ basop-ref-compat-encoder-ltv: - ivas-linux-fast variables: PARAM_FILE: scripts/config/self_test_ltv_basop_encoder.prm + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS script: - exit 0 @@ -537,6 +541,7 @@ basop-ref-compat-encoder-dmx-comp: PARAM_FILE: scripts/config/self_test_basop_encoder.prm # USING PYTEST_ADDOPTS env var did not work for some reason when testing locally - maybe because this is a custom option PYTEST_ARGS: "--compare_enc_dmx" + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS script: - exit 0 diff --git a/ci/remove_unsupported_testcases.py b/ci/remove_unsupported_testcases.py index e5e184fa83..667463ec7a 100644 --- a/ci/remove_unsupported_testcases.py +++ b/ci/remove_unsupported_testcases.py @@ -66,6 +66,8 @@ TESTCASES_MAIN = [ "MASA 1TC at 256kbps, 48kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out custom configuration", "MASA 1TC at 256 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out, HR custom configuration", "OMASA 2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out", + "Multi-channel 7_1_4 bitrate switching, 48kHz in, 48kHz out, BINAURAL out, HR, JBM Prof 5", + "Multi-channel 7_1 bitrate switching, 48kHz in, 32kHz out, BINAURAL_ROOM_REVERB out, HR, JBM Prof 5", # object editing starts here "OMASA 2Dir2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM", "OMASA 2Dir2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC", -- GitLab From 5f6b3922a80a7139913ded0da11f1582e275053e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 3 Apr 2025 13:59:21 +0200 Subject: [PATCH 37/46] add testcase to list of unsupported ones see https://forge.3gpp.org/rep/sa4/audio/ivas-basop/-/issues/1458 --- ci/remove_unsupported_testcases.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/remove_unsupported_testcases.py b/ci/remove_unsupported_testcases.py index 667463ec7a..870d513111 100644 --- a/ci/remove_unsupported_testcases.py +++ b/ci/remove_unsupported_testcases.py @@ -68,6 +68,7 @@ TESTCASES_MAIN = [ "OMASA 2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out", "Multi-channel 7_1_4 bitrate switching, 48kHz in, 48kHz out, BINAURAL out, HR, JBM Prof 5", "Multi-channel 7_1 bitrate switching, 48kHz in, 32kHz out, BINAURAL_ROOM_REVERB out, HR, JBM Prof 5", + "3 ISM with metadata bitrate switching from 48 kbps to 32 kbps, 48 kHz in, 32 kHz out, DTX, BINAURAL_ROOM_IR out, JBM Prof 5", # object editing starts here "OMASA 2Dir2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM", "OMASA 2Dir2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC", -- GitLab From d766538d6efe67d97fc4d42a5f4159199daeda34 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 10 Apr 2025 11:14:46 +0200 Subject: [PATCH 38/46] remove a testcase which is not compatible with the BASOP update branches --- ci/remove_unsupported_testcases.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/remove_unsupported_testcases.py b/ci/remove_unsupported_testcases.py index 870d513111..231b823648 100644 --- a/ci/remove_unsupported_testcases.py +++ b/ci/remove_unsupported_testcases.py @@ -102,6 +102,7 @@ TESTCASES_MAIN_PC = [ "OSBA 3OA 4ISM at 512 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out", "OSBA 3ISM 3OA at bitrate switching 13.2 to 512 kbps, 48kHz in, 32kHz out, BINAURAL ROOM REVERB out", "Multi-channel 5_1 bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 16kHz out, BINAURAL_ROOM_REVERB out", + "OSBA 2OA 2ISM bitrate switching, 32kHz in, 48kHz out, BINAURAL_ROOM_REVERB out, JBM Prof 5", ] -- GitLab From bfd9505699b72c948726d8352987798a44eddf88 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 11 Apr 2025 11:50:47 +0200 Subject: [PATCH 39/46] remove unsupported testcases from JBM neutral delay test --- tests/test_be_for_jbm_neutral_dly_profile.py | 25 +++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/tests/test_be_for_jbm_neutral_dly_profile.py b/tests/test_be_for_jbm_neutral_dly_profile.py index ec67ab22a3..53c585498c 100644 --- a/tests/test_be_for_jbm_neutral_dly_profile.py +++ b/tests/test_be_for_jbm_neutral_dly_profile.py @@ -18,7 +18,7 @@ DTX_OFF = "DTX_OFF" # testcases where DTX is supported TESTCASES_WITH_DTX = [ # stereo - ["stereo", 32000, "EXT"], + ["stereo", 32000, "STEREO"], ["stereo", 48000, "MONO"], ["stereo", 16400, "5_1"], ["stereo", 256000, "7_1_4"], @@ -40,8 +40,9 @@ TESTCASES_WITH_DTX = [ ["MASA1TC", 128000, "EXT"], # SBA ["HOA3", 64000, "BINAURAL"], - ["HOA2", 80000, "EXT"], - ["FOA", 13200, "stereo"], + ["HOA2", 80000, "HOA2"], + # NOTE: commented because still some bug in BASOP float reference + # ["FOA", 13200, "stereo"], ] # testcases with no DTX support @@ -53,27 +54,30 @@ TESTCASES_NO_DTX = [ # McMasa ["MC_5_1", 16400, "BINAURAL_ROOM_IR"], ["MC_7_1_4", 80000, "mono"], - ["MC_5_1_2", 24400, "EXT"], + ["MC_5_1_2", 24400, "5_1_2"], # paramMC ["MC_5_1_2", 48000, "BINAURAL"], - ["MC_7_1", 80000, "EXT"], + ["MC_7_1", 80000, "7_1"], ["MC_7_1_4", 128000, "FOA"], # paramUpmix ["MC_7_1_4", 160000, "stereo"], # discrete MC ["MC_5_1_2", 512000, "BINAURAL_ROOM_REVERB"], - ["MC_7_1", 128000, "EXT"], + ["MC_7_1", 128000, "7_1"], ["MC_7_1_4", 256000, "5_1"], # OMASA ["OMASA_ISM1", 512000, "BINAURAL"], ["OMASA_ISM2", 24400, "MONO"], ["OMASA_ISM3", 80000, "7_1_4"], ["OMASA_ISM4", 64000, "HOA3"], - ["OMASA_ISM2", 32000, "EXT"], + # NOTE: commented because EXT unsupported currently + # ["OMASA_ISM2", 32000, "EXT"], # OSBA ["OSBA_ISM2_HOA2", 64000, "BINAURAL_ROOM_IR"], - ["OSBA_ISM4_FOA", 512000, "BINAURAL_ROOM_REVERB"], - ["OSBA_ISM3_HOA3", 128000, "EXT"], + # NOTE: commented because still some bug in BASOP float reference + # ["OSBA_ISM4_FOA", 512000, "BINAURAL_ROOM_REVERB"], + # NOTE: commented because EXT unsupported currently + # ["OSBA_ISM3_HOA3", 128000, "EXT"], ["OSBA_ISM2_HOA3", 96000, "5_1"], ["OSBA_ISM1_HOA2", 32000, "mono"], ] @@ -181,6 +185,9 @@ def test_be_for_jbm_neutral_dly_profile_no_dtx( ) +@pytest.mark.skip( + reason="non-BE bug for DTX cases not yet fixed in BASOP ivas-float-update" +) @pytest.mark.parametrize("in_format,bitrate,out_format", TESTCASES_WITH_DTX) def test_be_for_jbm_neutral_dly_profile_with_dtx( in_format, -- GitLab From 91b0bce95d2b24db912a0283b4f1215f78994144 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 22 Apr 2025 15:28:46 +0200 Subject: [PATCH 40/46] Align test cases with BASOP CI, affects test_sba.py and test_renderer.py --- .gitlab-ci.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3fbaf04204..8f0405f84b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -32,6 +32,12 @@ variables: BASOP_REFERENCE_BRANCH: "ivas-float-update" SCALE_FACTOR: "3.162" PYTEST_ARGS: "" + LONG_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/renderer --param_file scripts/config/self_test_ltv.prm --use_ltv" + LONG_TEST_SUITE_NO_RENDERER: "tests/codec_be_on_mr_nonselection --param_file scripts/config/self_test_ltv.prm --use_ltv" + SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" + SHORT_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_basop_encoder.prm" + LONG_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_ltv_basop_encoder.prm" + TEST_SUITE: "" default: @@ -468,7 +474,7 @@ branch-is-up-to-date-with-main-post: tags: - ivas-linux before_script: - - python3 ci/remove_unsupported_testcases.py $PARAM_FILE + - python3 ci/remove_unsupported_testcases.py scripts/config/self_test.prm scripts/config/self_test_ltv.prm - python3 tests/create_short_testvectors.py - python3 scripts/prepare_combined_format_inputs.py @@ -477,9 +483,9 @@ branch-is-up-to-date-with-main-post: - *get-basop-float-reference - - python3 -m pytest tests/codec_be_on_mr_nonselection $PYTEST_ARGS --param_file $PARAM_FILE --update_ref 1 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$TESTCASE_TIMEOUT || true + - python3 -m pytest $TEST_SUITE $PYTEST_ARGS --update_ref 1 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$TESTCASE_TIMEOUT || true - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - if [ $zero_errors != 1 ]; then echo "Run errors encountered with $PARAM_FILE !"; exit $EXIT_CODE_FAIL; fi + - if [ $zero_errors != 1 ]; then echo "Run errors encountered with TEST_SUITE=$TEST_SUITE !"; exit $EXIT_CODE_FAIL; fi artifacts: name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" @@ -498,7 +504,7 @@ basop-ref-compat-stv: extends: - .basop-ci-branch-compat-template variables: - PARAM_FILE: scripts/config/self_test.prm + TEST_SUITE: $SHORT_TEST_SUITE TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS script: - exit 0 @@ -509,7 +515,7 @@ basop-ref-compat-ltv: tags: - ivas-linux-fast variables: - PARAM_FILE: scripts/config/self_test_ltv.prm + TEST_SUITE: $LONG_TEST_SUITE TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS script: - exit 0 @@ -518,7 +524,7 @@ basop-ref-compat-encoder-stv: extends: - .basop-ci-branch-compat-template variables: - PARAM_FILE: scripts/config/self_test_basop_encoder.prm + TEST_SUITE: $SHORT_TEST_SUITE_ENCODER TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS script: - exit 0 @@ -529,7 +535,7 @@ basop-ref-compat-encoder-ltv: tags: - ivas-linux-fast variables: - PARAM_FILE: scripts/config/self_test_ltv_basop_encoder.prm + TEST_SUITE: $LONG_TEST_SUITE_ENCODER TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS script: - exit 0 @@ -538,7 +544,7 @@ basop-ref-compat-encoder-dmx-comp: extends: - .basop-ci-branch-compat-template variables: - PARAM_FILE: scripts/config/self_test_basop_encoder.prm + TEST_SUITE: $SHORT_TEST_SUITE_ENCODER # USING PYTEST_ADDOPTS env var did not work for some reason when testing locally - maybe because this is a custom option PYTEST_ARGS: "--compare_enc_dmx" TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS -- GitLab From e02ef396e9cb8fbac526f346bbf76506444ec25e Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 22 Apr 2025 17:05:39 +0200 Subject: [PATCH 41/46] Include renderer tests in basop-ref-compat-stv --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8f0405f84b..18e5190116 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -504,7 +504,7 @@ basop-ref-compat-stv: extends: - .basop-ci-branch-compat-template variables: - TEST_SUITE: $SHORT_TEST_SUITE + TEST_SUITE: "$SHORT_TEST_SUITE tests/renderer" TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS script: - exit 0 -- GitLab From 6931121b1f054b5b58defab6188b1fc4e8e149be Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 23 Apr 2025 08:53:18 +0200 Subject: [PATCH 42/46] fix split rendering tests api change of cmp_pcm for the split comparison broke them --- tests/split_rendering/utils.py | 143 +++++++++++++++++++-------------- 1 file changed, 82 insertions(+), 61 deletions(-) diff --git a/tests/split_rendering/utils.py b/tests/split_rendering/utils.py index 53de170c91..f9a1e86fc7 100644 --- a/tests/split_rendering/utils.py +++ b/tests/split_rendering/utils.py @@ -1,33 +1,33 @@ #!/usr/bin/env python3 """ - (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +(C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ import sys @@ -39,7 +39,13 @@ from typing import Tuple import numpy as np import pytest -from tests.renderer.utils import check_BE, run_ivas_isar_enc_cmd, run_ivas_isar_dec_cmd, run_isar_post_rend_cmd, run_isar_ext_rend_cmd +from tests.renderer.utils import ( + check_BE, + run_ivas_isar_enc_cmd, + run_ivas_isar_dec_cmd, + run_isar_post_rend_cmd, + run_isar_ext_rend_cmd, +) from tests.split_rendering.constants import * from ..cmp_pcm import cmp_pcm from ..conftest import parse_properties @@ -62,8 +68,10 @@ def lc3plus_used(test_info, in_fmt, render_config): or ("default" in render_config and "external_split" in test_info.node.name) ) + FRAME_SIZE_RE = re.compile(r"(\d+)ms") + def transport_codec_frame_size_ms(test_info, in_fmt, render_config, pre_rend_fr): explicit_frame_size_ms = FRAME_SIZE_RE.search(render_config) @@ -154,7 +162,7 @@ def truncate_signal( def run_full_chain_split_rendering( record_property, - props_to_record, + props_to_record, test_info, in_fmt: str, bitrate: str, @@ -166,29 +174,31 @@ def run_full_chain_split_rendering( post_rend_fr: str = "20", pre_rend_fr: str = "20", get_mld=False, - mld_lim=0, + mld_lim=0, get_ssnr=False, get_odg=False, - get_odg_bin=False, + get_odg_bin=False, ) -> str: """ Runs the full split rendering chain consisting of the IVAS encoder, decoder and split renderer """ - check_xfail(test_info, in_fmt, render_config.name, pre_rend_fr, post_rend_fr, bitrate) + check_xfail( + test_info, in_fmt, render_config.name, pre_rend_fr, post_rend_fr, bitrate + ) with TemporaryDirectory() as tmp_dir: tmp_dir = Path(tmp_dir) cut_in_file = tmp_dir.joinpath("cut_input.wav") - #ivas_bitstream = tmp_dir.joinpath("ivas.192") + # ivas_bitstream = tmp_dir.joinpath("ivas.192") ivas_bitstream_stem = f"{in_fmt}_{bitrate}bps_{renderer_fmt}_split_full_config_{render_config.stem}_prerfr_{pre_rend_fr}_postrfr_{post_rend_fr}_ivas.192" - #split_bitstream = tmp_dir.joinpath("split.bit") + # split_bitstream = tmp_dir.joinpath("split.bit") split_bitstream_stem = f"{in_fmt}_{bitrate}bps_{renderer_fmt}_split_full_config_{render_config.stem}_prerfr_{pre_rend_fr}_postrfr_{post_rend_fr}_split.bit" if renderer_fmt == "BINAURAL_SPLIT_PCM": - #split_md_file = tmp_dir.joinpath("split_md.bin") + # split_md_file = tmp_dir.joinpath("split_md.bin") split_md_file_stem = f"{in_fmt}_{bitrate}bps_{renderer_fmt}_split_full_config_{render_config.stem}_prerfr_{pre_rend_fr}_postrfr_{post_rend_fr}_split_md.bit" - + out_file_stem = f"{in_fmt}_{bitrate}bps_{renderer_fmt}_split_full_config_{render_config.stem}_prerfr_{pre_rend_fr}_postrfr_{post_rend_fr}_.wav" if test_info.config.option.create_ref: @@ -206,7 +216,11 @@ def run_full_chain_split_rendering( if in_fmt.upper().startswith("OSBA"): # use same MD as ISM in_meta_files = FORMAT_TO_METADATA_FILES[f"ISM{in_fmt[5]}"] - elif in_fmt.upper().startswith("ISM") or in_fmt.upper().startswith("MASA") or in_fmt.upper().startswith("OMASA"): + elif ( + in_fmt.upper().startswith("ISM") + or in_fmt.upper().startswith("MASA") + or in_fmt.upper().startswith("OMASA") + ): in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] else: in_meta_files = None @@ -256,7 +270,7 @@ def run_full_chain_split_rendering( # run split renderer cmd = SPLIT_POST_REND_CMD[:] - #if test_info.config.option.create_ref: + # if test_info.config.option.create_ref: # cmd[0] += BIN_SUFFIX_MERGETARGET cmd[0] += binary_suffix cmd[4] = str(split_bitstream) @@ -283,35 +297,38 @@ def run_full_chain_split_rendering( cut, cut_fs = readfile(out_file) - if get_mld == False: - [diff_found, snr, gain_b, max_diff] = check_BE(test_info, ref, ref_fs, cut, cut_fs) - if diff_found : + if not get_mld: + [diff_found, snr, gain_b, max_diff] = check_BE( + test_info, ref, ref_fs, cut, cut_fs + ) + if diff_found: pytest.fail( f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" ) else: - # see constants.py output_differs, reason = cmp_pcm( out_file, out_file_ref, - "BINAURAL", + 2, # is always "BINAURAL" ref_fs, get_mld=get_mld, mld_lim=mld_lim, get_ssnr=get_ssnr, get_odg=get_odg, - get_odg_bin=get_odg_bin, + get_odg_bin=get_odg_bin, ) - + + # NOTE: split comparison not implemented for split rendering, always unpack values + reason = reason[0] + output_differs = output_differs[0] + props = parse_properties(reason, output_differs, props_to_record) for k, v in props.items(): record_property(k, v) - + if output_differs: pytest.fail(f"Output differs: ({reason})") - - - + return out_file @@ -330,10 +347,10 @@ def run_external_split_rendering( post_rend_fr: str = "20", pre_rend_fr: str = "20", get_mld=False, - mld_lim=0, + mld_lim=0, get_ssnr=False, get_odg=False, - get_odg_bin=False, + get_odg_bin=False, ) -> Tuple[np.ndarray, int]: """ Runs the exeternal split rendering chain consisting of @@ -399,7 +416,7 @@ def run_external_split_rendering( # run split renderer cmd = SPLIT_POST_REND_CMD[:] - #if test_info.config.option.create_ref: + # if test_info.config.option.create_ref: # cmd[0] += BIN_SUFFIX_MERGETARGET cmd[0] += binary_suffix cmd[4] = str(split_bitstream) @@ -434,8 +451,10 @@ def run_external_split_rendering( cut, cut_fs = readfile(out_file) if get_mld == False: - [diff_found, snr, gain_b, max_diff] = check_BE(test_info, ref, ref_fs, cut, cut_fs) - if diff_found : + [diff_found, snr, gain_b, max_diff] = check_BE( + test_info, ref, ref_fs, cut, cut_fs + ) + if diff_found: pytest.fail( f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" ) @@ -444,21 +463,23 @@ def run_external_split_rendering( output_differs, reason = cmp_pcm( out_file, out_file_ref, - "BINAURAL", + 2, # is always "BINAURAL", ref_fs, get_mld=get_mld, mld_lim=mld_lim, get_ssnr=get_ssnr, get_odg=get_odg, - get_odg_bin=get_odg_bin, + get_odg_bin=get_odg_bin, ) - + # NOTE: split comparison not implemented for split rendering, always unpack values + reason = reason[0] + output_differs = output_differs[0] + props = parse_properties(reason, output_differs, props_to_record) for k, v in props.items(): record_property(k, v) - + if output_differs: - pytest.fail(f"Output differs: ({reason})") - - + pytest.fail(f"Output differs: ({reason})") + return out_file -- GitLab From 5e50eca57b6ef793d80ec060bee3e2fed91c8e4f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 23 Apr 2025 09:34:55 +0200 Subject: [PATCH 43/46] revert basop-specific changes to JBM BE test --- tests/test_be_for_jbm_neutral_dly_profile.py | 25 +++++++------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/tests/test_be_for_jbm_neutral_dly_profile.py b/tests/test_be_for_jbm_neutral_dly_profile.py index 53c585498c..ec67ab22a3 100644 --- a/tests/test_be_for_jbm_neutral_dly_profile.py +++ b/tests/test_be_for_jbm_neutral_dly_profile.py @@ -18,7 +18,7 @@ DTX_OFF = "DTX_OFF" # testcases where DTX is supported TESTCASES_WITH_DTX = [ # stereo - ["stereo", 32000, "STEREO"], + ["stereo", 32000, "EXT"], ["stereo", 48000, "MONO"], ["stereo", 16400, "5_1"], ["stereo", 256000, "7_1_4"], @@ -40,9 +40,8 @@ TESTCASES_WITH_DTX = [ ["MASA1TC", 128000, "EXT"], # SBA ["HOA3", 64000, "BINAURAL"], - ["HOA2", 80000, "HOA2"], - # NOTE: commented because still some bug in BASOP float reference - # ["FOA", 13200, "stereo"], + ["HOA2", 80000, "EXT"], + ["FOA", 13200, "stereo"], ] # testcases with no DTX support @@ -54,30 +53,27 @@ TESTCASES_NO_DTX = [ # McMasa ["MC_5_1", 16400, "BINAURAL_ROOM_IR"], ["MC_7_1_4", 80000, "mono"], - ["MC_5_1_2", 24400, "5_1_2"], + ["MC_5_1_2", 24400, "EXT"], # paramMC ["MC_5_1_2", 48000, "BINAURAL"], - ["MC_7_1", 80000, "7_1"], + ["MC_7_1", 80000, "EXT"], ["MC_7_1_4", 128000, "FOA"], # paramUpmix ["MC_7_1_4", 160000, "stereo"], # discrete MC ["MC_5_1_2", 512000, "BINAURAL_ROOM_REVERB"], - ["MC_7_1", 128000, "7_1"], + ["MC_7_1", 128000, "EXT"], ["MC_7_1_4", 256000, "5_1"], # OMASA ["OMASA_ISM1", 512000, "BINAURAL"], ["OMASA_ISM2", 24400, "MONO"], ["OMASA_ISM3", 80000, "7_1_4"], ["OMASA_ISM4", 64000, "HOA3"], - # NOTE: commented because EXT unsupported currently - # ["OMASA_ISM2", 32000, "EXT"], + ["OMASA_ISM2", 32000, "EXT"], # OSBA ["OSBA_ISM2_HOA2", 64000, "BINAURAL_ROOM_IR"], - # NOTE: commented because still some bug in BASOP float reference - # ["OSBA_ISM4_FOA", 512000, "BINAURAL_ROOM_REVERB"], - # NOTE: commented because EXT unsupported currently - # ["OSBA_ISM3_HOA3", 128000, "EXT"], + ["OSBA_ISM4_FOA", 512000, "BINAURAL_ROOM_REVERB"], + ["OSBA_ISM3_HOA3", 128000, "EXT"], ["OSBA_ISM2_HOA3", 96000, "5_1"], ["OSBA_ISM1_HOA2", 32000, "mono"], ] @@ -185,9 +181,6 @@ def test_be_for_jbm_neutral_dly_profile_no_dtx( ) -@pytest.mark.skip( - reason="non-BE bug for DTX cases not yet fixed in BASOP ivas-float-update" -) @pytest.mark.parametrize("in_format,bitrate,out_format", TESTCASES_WITH_DTX) def test_be_for_jbm_neutral_dly_profile_with_dtx( in_format, -- GitLab From 82881a810d2e8ae27440d60b72648979c1ed43b5 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 23 Apr 2025 09:35:32 +0200 Subject: [PATCH 44/46] revert deletion of readme file --- ...end_sofa_to_rom_table_converter_readme.txt | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 scripts/binauralRenderer_interface/ivas_crend_sofa_to_rom_table_converter_readme.txt diff --git a/scripts/binauralRenderer_interface/ivas_crend_sofa_to_rom_table_converter_readme.txt b/scripts/binauralRenderer_interface/ivas_crend_sofa_to_rom_table_converter_readme.txt new file mode 100644 index 0000000000..c3675ab45a --- /dev/null +++ b/scripts/binauralRenderer_interface/ivas_crend_sofa_to_rom_table_converter_readme.txt @@ -0,0 +1,82 @@ +/****************************************************************************************************** + + (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository. All Rights Reserved. + + This software is protected by copyright law and by international treaties. + The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository retain full ownership rights in their respective contributions in + the software. This notice grants no license of any kind, including but not limited to patent + license, nor is any license granted by implication, estoppel or otherwise. + + Contributors are required to enter into the IVAS codec Public Collaboration agreement before making + contributions. + + This software is provided "AS IS", without any express or implied warranties. The software is in the + development stage. It is intended exclusively for experts who have experience with such software and + solely for the purpose of inspection. All implied warranties of non-infringement, merchantability + and fitness for a particular purpose are hereby disclaimed and excluded. + + Any dispute, controversy or claim arising under or in relation to providing this software shall be + submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in + accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and + the United Nations Convention on Contracts on the International Sales of Goods. + +*******************************************************************************************************/ + +Mixer conv sofa to rom table or binary files converter program +------------------------------ + +The sofa to rom format or binary files converter is used to generate the c files (*.h,*.c) or binary files containing the binary representation of the binaural +filters for renderer MIXER_CONV and MIXER_CONV_ROOM + +First, build the converter under scripts/binauralRenderer_interface in VSCode (using CMakeList) or using the following command lines : + - mkdir build + - cd build + - cmake .. + - cmake --build . (option --config=release) + - cd .. + - Test : + - run py run.py (require matplotlib) -> no error and a figure with impulse response shall appear + +Requirements: +------------ + - Python version 3.9.* with pip + - numpy (pip install numpy) + - netCDF4 (pip install netCDF4) + - SofaReader.py file must be in the same folder as the executable generate_crend_ivas_tables + +Usage: +------ +generate_crend_ivas_tables [Options] + +Mandatory parameters : + - Frame size in ms (5 or 20) + - list of sofa files at least one. For rom files generation it is needed : + - one with HRIR with directions of IVAS combined speakers configuration or more + - one with FOA to binaural filters, can be obtain from HRIR using ./matlab_hrir_generation_scripts/convert_SD2SHD_HRIRs.m matlab script + - one with HOA order 2 to binaural filters, can be obtain from HRIR using ./matlab_hrir_generation_scripts/convert_SD2SHD_HRIRs.m matlab script + - one with HOA order 3 to binaural filters, can be obtain from HRIR using ./matlab_hrir_generation_scripts/convert_SD2SHD_HRIRs.m matlab script + - one with BRIR filters that directions of IVAS combined speakers configuration or more + +Options : + -lib_rend_path : path where header and c files will be created (if missing rom files not generated). + -binary_files_path : path where binary files will be created (if missing binary files not generated)\n + -binary_common_file_name : common name for binary files that will be created (crend_hrir-53_brir-Fhg for example + -compute_reverb_rom sofa_file_path : if present reverb rom tables and binary files are computed using sofa_file_path as input HRIRs + -brir_optim_config_path config_file_path : if present brir optimisation process uses parameters stored in config_file_path + Configuration example files are provided in folder : + -brir_no_optim.cfg : for exact convolution of the BRIR by IVAS decoder or renderer + -brir_default_optim.cfg : default values BRIR optimisation used by IVAS decoder or renderer + -brir_low_complexity_optim.cfg : values for BRIR optimisation reducing IVAS decoder or renderer complexity + +For example : + +.\\generate_crend_ivas_tables.exe -lib_rend_path ../../lib_rend 5 ./HRIRs_sofa/HRIR_128_Meth5_IRC_53_Q10_symL_Itrp1_48000.sofa ./HRIRs_sofa/HRIR_128_Meth5_IRC_53_Q10_symL_Itrp1_48000_FOA.sofa ./HRIRs_sofa/HRIR_128_Meth5_IRC_53_Q10_symL_Itrp1_48000_HOA2.sofa ./HRIRs_sofa/HRIR_128_Meth5_IRC_53_Q10_symL_Itrp1_48000_HOA3.sofa ./BRIRs_sofa/IIS_BRIR_officialMPEG_Combined.sofa +./generate_crend_ivas_tables -lib_rend_path ../../lib_rend 5 ./HRIRs_sofa/HRIR_128_Meth5_IRC_53_Q10_symL_Itrp1_48000.sofa ./HRIRs_sofa/HRIR_128_Meth5_IRC_53_Q10_symL_Itrp1_48000_FOA.sofa ./HRIRs_sofa/HRIR_128_Meth5_IRC_53_Q10_symL_Itrp1_48000_HAO2.sofa ./HRIRs_sofa/HRIR_128_Meth5_IRC_53_Q10_symL_Itrp1_48000_HOA3.sofa ./BRIRs_sofa/IIS_BRIR_officialMPEG_Combined.sofa -- GitLab From db972abf523ba5b8e51f875943111d1ae35c3438 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 23 Apr 2025 10:43:12 +0200 Subject: [PATCH 45/46] increase per-testcase timeout for stv sanitizer test --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c3fbcc0b1b..e7a8c30475 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,7 +27,7 @@ variables: - 'test-branch-vs-input-passthrough' GIT_CLEAN_FLAGS: -ffdxq - TESTCASE_TIMEOUT_STV_SANITIZERS: 180 + TESTCASE_TIMEOUT_STV_SANITIZERS: 240 TESTCASE_TIMEOUT_LTV_SANITIZERS: 2400 BASOP_REFERENCE_BRANCH: "ivas-float-update" SCALE_FACTOR: "3.162" -- GitLab From c82d4dadb52390a513fde94edcea5e798573e0a8 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 23 Apr 2025 12:21:07 +0200 Subject: [PATCH 46/46] use default separator (",") for csv files --- scripts/basop_check_for_changes_in_testcases.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index 2b38e8a0d5..3a61c76bb4 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -42,7 +42,7 @@ import xml.etree.ElementTree as ET # set positive threshold for "lower is better" metrics, negative for "higher is better" COLS_2_THRESHOLDS = { "MLD": float(os.environ.get("CI_REGRESSION_THRESH_MLD", 0.1)), - "MAXIMUM ABS DIFF": float(os.environ.get("CI_REGRESSION_THRESH_MAX_ABS_DIFF", 5)), + "MAX_ABS_DIFF": float(os.environ.get("CI_REGRESSION_THRESH_MAX_ABS_DIFF", 5)), "MIN_SSNR": float(os.environ.get("CI_REGRESSION_THRESH_SSNR", -1)), "MIN_ODG": float(os.environ.get("CI_REGRESSION_THRESH_ODG", -0.05)), } @@ -62,8 +62,8 @@ PATTERNS = [PATTERN_ENC, PATTERN_DEC, PATTERN_EID, PATTERN_NETSIM] def main(args): xml_report = args.xml_report - df_curr = pd.read_csv(args.csv_current, sep=";") - df_prev = pd.read_csv(args.csv_previous, sep=";") + df_curr = pd.read_csv(args.csv_current) + df_prev = pd.read_csv(args.csv_previous) df_merged = pd.merge(df_curr, df_prev, on="testcase", suffixes=["-curr", "-prev"]) # remove leading path from testcase names for better readability @@ -76,8 +76,8 @@ def main(args): regressions_found = False # check for newly introduced crashes - col_curr = "Result-curr" - col_prev = "Result-prev" + col_curr = "result-curr" + col_prev = "result-prev" mask_crash_introduced = (df_merged[col_curr] == "ERROR") & ( df_merged[col_prev] != "ERROR" ) -- GitLab