From 71731848cc3c409659b4dea2be75643f38855625 Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Thu, 6 Jun 2024 12:33:53 +0200
Subject: [PATCH 01/30] add more modes in voip be test and use all output
 formats

---
 ci/ivas_voip_be_test.sh | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/ci/ivas_voip_be_test.sh b/ci/ivas_voip_be_test.sh
index 7c5c834419..4470352924 100755
--- a/ci/ivas_voip_be_test.sh
+++ b/ci/ivas_voip_be_test.sh
@@ -60,8 +60,26 @@ if [ $BUILD -eq 1 ];then
 fi
 
 # Configuration
-modes=('HOA3_b128_wb_cbr' 'MC_7_1_b96_fb_cbr' 'ISM2_b48_fb_cbr')
-output_formats=('STEREO' 'FOA' '7_1' 'HOA3')
+modes=(
+    'stereo_b32_dtx_wb_cbr'
+    'stereo_b48_dtx_fb_cbr'
+    'ISM4_b48_dtx_fb_cbr'
+    'ISM3_b64_dtx_swb_cbr'
+    'MASA_1TC_b24_4_dtx_wb_cbr'
+    'MASA_2TC_b80_dtx_fb_cbr'
+    'MC_5_1_b128_swb_cbr'
+    'MC_7_1_b96_wb_cbr'
+    'MC_5_1_4_b48_fb_cbr'
+    'MC_7_1_4_b160_fb_cbr'
+    'OMASA_ISM1_2TC_b512_swb_cbr'
+    'OMASA_ISM2_1TC_b24_4_wb_cbr'
+    'OMASA_ISM3_2TC_b80_fb_cbr'
+    'OMASA_ISM4_1TC_b48_swb_cbr'
+    'OSBA_ISM4_HOA2_b512_fb_cbr'
+    'OSBA_ISM2_HOA2_b64_wb_cbr'
+    'SBA_HOA3_b64_swb_cbr'
+    'SBA_FOA_b256_fb_cbr'
+)
 limit_input_to_x_seconds=30
 verbosity_cmd="-z console"
 
@@ -72,8 +90,8 @@ output_dir_default="out"
 output_dir_voip="out_voip"
 
 # Run the same modes in VoIP and non-VoIP mode with a neutral delay profile
-./scripts/runIvasCodec.py $verbosity_cmd -p $cfg $WORKERS -U $limit_input_to_x_seconds -m "${modes[@]}" --oc "${output_formats[@]}" -o $output_dir_default | tee voip_be_test_output.txt
-./scripts/runIvasCodec.py $verbosity_cmd -p $cfg $WORKERS -U $limit_input_to_x_seconds -m "${modes[@]}" --oc "${output_formats[@]}" -o $output_dir_voip -J "$dly_profile" | tee -a voip_be_test_output.txt
+./scripts/runIvasCodec.py $verbosity_cmd -p $cfg $WORKERS -U $limit_input_to_x_seconds -m "${modes[@]}" -o $output_dir_default | tee voip_be_test_output.txt
+./scripts/runIvasCodec.py $verbosity_cmd -p $cfg $WORKERS -U $limit_input_to_x_seconds -m "${modes[@]}" -o $output_dir_voip -J "$dly_profile" --decoder_only | tee -a voip_be_test_output.txt
 
 # Check if Python scripts above failed. They return status 0 even when running a mode fails, so we have to parse log file
 if grep -iq failed voip_be_test_output.txt ; then
-- 
GitLab


From 035d66bec2d342df099b602c840a4c1b25e82e9d Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Thu, 6 Jun 2024 13:48:58 +0200
Subject: [PATCH 02/30] correct mode strings and copy bitstreams to voip dir

---
 ci/ivas_voip_be_test.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/ci/ivas_voip_be_test.sh b/ci/ivas_voip_be_test.sh
index 4470352924..1deb12411d 100755
--- a/ci/ivas_voip_be_test.sh
+++ b/ci/ivas_voip_be_test.sh
@@ -77,8 +77,8 @@ modes=(
     'OMASA_ISM4_1TC_b48_swb_cbr'
     'OSBA_ISM4_HOA2_b512_fb_cbr'
     'OSBA_ISM2_HOA2_b64_wb_cbr'
-    'SBA_HOA3_b64_swb_cbr'
-    'SBA_FOA_b256_fb_cbr'
+    'HOA3_b64_swb_cbr'
+    'FOA_b256_fb_cbr'
 )
 limit_input_to_x_seconds=30
 verbosity_cmd="-z console"
@@ -91,6 +91,8 @@ output_dir_voip="out_voip"
 
 # Run the same modes in VoIP and non-VoIP mode with a neutral delay profile
 ./scripts/runIvasCodec.py $verbosity_cmd -p $cfg $WORKERS -U $limit_input_to_x_seconds -m "${modes[@]}" -o $output_dir_default | tee voip_be_test_output.txt
+mkdir $output_dir_voip
+cp -r $output_dir_default/enc $output_dir_voip/enc
 ./scripts/runIvasCodec.py $verbosity_cmd -p $cfg $WORKERS -U $limit_input_to_x_seconds -m "${modes[@]}" -o $output_dir_voip -J "$dly_profile" --decoder_only | tee -a voip_be_test_output.txt
 
 # Check if Python scripts above failed. They return status 0 even when running a mode fails, so we have to parse log file
-- 
GitLab


From 3c6a69883be2672d0b452a75846742702ba0991e Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Thu, 6 Jun 2024 14:51:56 +0200
Subject: [PATCH 03/30] use parallelized script for comparing files

---
 ci/ivas_voip_be_test.sh     | 33 ++++-----------------------------
 scripts/batch_comp_audio.py |  4 +++-
 2 files changed, 7 insertions(+), 30 deletions(-)

diff --git a/ci/ivas_voip_be_test.sh b/ci/ivas_voip_be_test.sh
index 1deb12411d..9ffcd367e8 100755
--- a/ci/ivas_voip_be_test.sh
+++ b/ci/ivas_voip_be_test.sh
@@ -119,43 +119,18 @@ if [[ ! -d $output_dir_voip_dec_trimmed ]]; then
     mkdir $output_dir_voip_dec_trimmed 
 fi
 
+# trim and rename in one go
+# renaming is necessary for batch_comp_audio later
 for cut in "$output_dir_voip_dec"/*.wav; do
     output_path=${cut/$output_dir_voip_dec/$output_dir_voip_dec_trimmed}
-    output_path=${output_path/".wav"/".raw"}
+    output_path=${output_path/_jbm_dly_error_profile_0_dat/}
     python3 "$python_audiofile_script_path" pre-trim 60 "$cut" "$output_path" | tee -a voip_be_test_output.txt
 done
 
-# Convert non-VoIP output from wav to pcm (comparison script doesn't support wav)
-output_dir_default_dec="$output_dir_default"/dec
-output_dir_default_dec_pcm="$output_dir_default"/dec_pcm
-
-if [[ ! -d $output_dir_default_dec_pcm ]]; then
-    mkdir $output_dir_default_dec_pcm 
-fi
-
-for ref in "$output_dir_default_dec"/*.wav; do
-    output_path=${ref/$output_dir_default_dec/$output_dir_default_dec_pcm}
-    output_path=${output_path/".wav"/".raw"}
-    python3 "$python_audiofile_script_path" convert "$ref" "$output_path" | tee -a voip_be_test_output.txt
-done
-
 # Assert BE between non-VoIP and VoIP modes
 all_be=1
 
-cmp_tool_path=$(pwd)/tests/cmp_pcm.py
-
-for ref in "$output_dir_default_dec_pcm"/*; do
-    cut=${ref/$output_dir_default_dec_pcm/$output_dir_voip_dec_trimmed}
-    cut=${cut/".dec."/"_jbm_dly_error_profile_0_dat.dec."}
-
-    # Print paths of compared files, since the script doesn't do it
-    printf "\nComparing %s and %s\n" "$ref" "$cut" | tee -a voip_be_test_output.txt
-    printout=$($cmp_tool_path "$ref" "$cut")
-    if [ $? -ne 0 ]; then
-        all_be=0
-    fi
-    printf "%s\n" "$printout" | tee -a voip_be_test_output.txt
-done
+$(pwd)/scripts/batch_comp_audio.py "$output_dir_voip_dec_trimmed"/dec "$output_dir_default"/dec
 
 if [ $all_be -eq 1 ]; then
     printf "\n\nAll tested conditions are bit-exact\n" | tee -a voip_be_test_output.txt
diff --git a/scripts/batch_comp_audio.py b/scripts/batch_comp_audio.py
index ce1f380c56..cfc546f222 100755
--- a/scripts/batch_comp_audio.py
+++ b/scripts/batch_comp_audio.py
@@ -133,6 +133,8 @@ def main(args):
         else:
             print("All files are bitexact")
 
+    return num_files_diff
+
 
 def compare_files(f, fol1, fol2, outputs_dict, tool):
     """
@@ -290,4 +292,4 @@ if __name__ == "__main__":
     )
     args = parser.parse_args()
 
-    main(args)
+    sys.exit(main(args))
-- 
GitLab


From 69b28dd90f284d1a7569ee9ae1f597c9e802e090 Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Thu, 6 Jun 2024 14:59:54 +0200
Subject: [PATCH 04/30] fix fail detection and add dbg output

---
 ci/ivas_voip_be_test.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ci/ivas_voip_be_test.sh b/ci/ivas_voip_be_test.sh
index 9ffcd367e8..ea4546b59e 100755
--- a/ci/ivas_voip_be_test.sh
+++ b/ci/ivas_voip_be_test.sh
@@ -121,18 +121,18 @@ fi
 
 # trim and rename in one go
 # renaming is necessary for batch_comp_audio later
+echo "Trim JBM delay from voip files"
 for cut in "$output_dir_voip_dec"/*.wav; do
     output_path=${cut/$output_dir_voip_dec/$output_dir_voip_dec_trimmed}
     output_path=${output_path/_jbm_dly_error_profile_0_dat/}
+    echo "$output_path\n"
     python3 "$python_audiofile_script_path" pre-trim 60 "$cut" "$output_path" | tee -a voip_be_test_output.txt
 done
 
 # Assert BE between non-VoIP and VoIP modes
-all_be=1
+num_diff=$(./scripts/batch_comp_audio.py "$output_dir_voip_dec_trimmed"/dec "$output_dir_default"/dec --tool mld)
 
-$(pwd)/scripts/batch_comp_audio.py "$output_dir_voip_dec_trimmed"/dec "$output_dir_default"/dec
-
-if [ $all_be -eq 1 ]; then
+if [ $num_diff -ne 0 ]; then
     printf "\n\nAll tested conditions are bit-exact\n" | tee -a voip_be_test_output.txt
 else
     printf "\n\nBitexactness problems found!\n" | tee -a voip_be_test_output.txt
-- 
GitLab


From da13fbe0f35582feff457e84074cb6c3754c8cc5 Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Thu, 6 Jun 2024 15:16:36 +0200
Subject: [PATCH 05/30] add parallelization for trimming

---
 ci/ivas_voip_be_test.sh | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/ci/ivas_voip_be_test.sh b/ci/ivas_voip_be_test.sh
index ea4546b59e..1e48219d35 100755
--- a/ci/ivas_voip_be_test.sh
+++ b/ci/ivas_voip_be_test.sh
@@ -126,13 +126,19 @@ for cut in "$output_dir_voip_dec"/*.wav; do
     output_path=${cut/$output_dir_voip_dec/$output_dir_voip_dec_trimmed}
     output_path=${output_path/_jbm_dly_error_profile_0_dat/}
     echo "$output_path\n"
-    python3 "$python_audiofile_script_path" pre-trim 60 "$cut" "$output_path" | tee -a voip_be_test_output.txt
+    python3 "$python_audiofile_script_path" pre-trim 60 "$cut" "$output_path" | tee -a voip_be_test_output.txt &
+done
+
+# wait for all trims to be done
+for job in `jobs -p`; do
+    wait $job
 done
 
 # Assert BE between non-VoIP and VoIP modes
-num_diff=$(./scripts/batch_comp_audio.py "$output_dir_voip_dec_trimmed"/dec "$output_dir_default"/dec --tool mld)
+num_diff=$(./scripts/batch_comp_audio.py "$output_dir_voip_dec_trimmed" "$output_dir_default"/dec --tool mld)
 
-if [ $num_diff -ne 0 ]; then
+echo "num diff: $num_diff"
+if [ "$num_diff" != "0" ]; then
     printf "\n\nAll tested conditions are bit-exact\n" | tee -a voip_be_test_output.txt
 else
     printf "\n\nBitexactness problems found!\n" | tee -a voip_be_test_output.txt
-- 
GitLab


From 6acd238f5a5cf528a2cf5874c036f60b4e2a07b0 Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Thu, 6 Jun 2024 15:54:48 +0200
Subject: [PATCH 06/30] add new command for comparing files and shorten voip
 test scritp

---
 ci/ivas_voip_be_test.sh              | 37 +++++++++++-----------------
 scripts/pyaudio3dtools/audioarray.py |  3 ++-
 scripts/pyaudio3dtools/audiofile.py  | 20 +++++++++++++++
 3 files changed, 37 insertions(+), 23 deletions(-)

diff --git a/ci/ivas_voip_be_test.sh b/ci/ivas_voip_be_test.sh
index 1e48219d35..5a2e7f9beb 100755
--- a/ci/ivas_voip_be_test.sh
+++ b/ci/ivas_voip_be_test.sh
@@ -111,34 +111,27 @@ python_audio_module_path=$(pwd)/scripts
 export PYTHONPATH=$python_audio_module_path:$PYTHONPATH
 python_audiofile_script_path=$python_audio_module_path/pyaudio3dtools/audiofile.py
 
-# Trim JBM delay from VoIP output files
-output_dir_voip_dec="$output_dir_voip"/dec
-output_dir_voip_dec_trimmed="$output_dir_voip"/dec_trimmed
+jbm_offset=60
+output_dir_default_dec="$output_dir_default"/dec
 
-if [[ ! -d $output_dir_voip_dec_trimmed ]]; then
-    mkdir $output_dir_voip_dec_trimmed 
-fi
+all_be=1
+pids=""
 
-# trim and rename in one go
-# renaming is necessary for batch_comp_audio later
-echo "Trim JBM delay from voip files"
-for cut in "$output_dir_voip_dec"/*.wav; do
-    output_path=${cut/$output_dir_voip_dec/$output_dir_voip_dec_trimmed}
-    output_path=${output_path/_jbm_dly_error_profile_0_dat/}
-    echo "$output_path\n"
-    python3 "$python_audiofile_script_path" pre-trim 60 "$cut" "$output_path" | tee -a voip_be_test_output.txt &
-done
+for ref in "$output_dir_default_dec"/*.wav; do
+    cut=${ref/$output_dir_default_dec/$output_dir_voip}
+    cut=${cut/".dec."/"_jbm_dly_error_profile_0_dat.dec."}
 
-# wait for all trims to be done
-for job in `jobs -p`; do
-    wait $job
+    # Print paths of compared files, since the script doesn't do it
+    printf "\nComparing %s and %s\n" "$ref" "$cut" | tee -a voip_be_test_output.txt
+    $python_audiofile_script_path compare "$ref" "$cut" "$jbm_offset" &
+    pids="$pids $!"
 done
 
-# Assert BE between non-VoIP and VoIP modes
-num_diff=$(./scripts/batch_comp_audio.py "$output_dir_voip_dec_trimmed" "$output_dir_default"/dec --tool mld)
+for pid in $pids; do
+    wait $pid || let "all_be=0"
+done
 
-echo "num diff: $num_diff"
-if [ "$num_diff" != "0" ]; then
+if [ $all_be -eq 1 ]; then
     printf "\n\nAll tested conditions are bit-exact\n" | tee -a voip_be_test_output.txt
 else
     printf "\n\nBitexactness problems found!\n" | tee -a voip_be_test_output.txt
diff --git a/scripts/pyaudio3dtools/audioarray.py b/scripts/pyaudio3dtools/audioarray.py
index d89c079d89..0da5238ca2 100644
--- a/scripts/pyaudio3dtools/audioarray.py
+++ b/scripts/pyaudio3dtools/audioarray.py
@@ -232,6 +232,7 @@ def compare(
     fs: int,
     per_frame: bool = True,
     get_mld: bool = False,
+    offset_test_start: int = 0,
 ) -> dict:
     """Compare two audio arrays
 
@@ -254,7 +255,7 @@ def compare(
         Comparison results
     """
     framesize = fs // 50
-    diff = abs(test - ref)
+    diff = abs(test[offset_test_start:, :] - ref)
     max_diff = int(diff.max())
     result = {
         "bitexact": True,
diff --git a/scripts/pyaudio3dtools/audiofile.py b/scripts/pyaudio3dtools/audiofile.py
index e614bfbd49..405a60d870 100644
--- a/scripts/pyaudio3dtools/audiofile.py
+++ b/scripts/pyaudio3dtools/audiofile.py
@@ -780,5 +780,25 @@ if __name__ == "__main__":
     parser_convert.add_argument("output_file")
     parser_convert.set_defaults(func=convert_wrapper)
 
+    def compare_wrapper(compare_args):
+        if not compare_args.ref_file.endswith(".wav") or not compare_args.test_file.endswith(".wav"):
+            print("Convert currently only supported with WAV file input")
+            exit(-1)
+
+        s1, fs1 = readfile(compare_args.ref_file, outdtype="int16")
+        s2, fs2 = readfile(compare_args.test_file, outdtype="int16")
+        if fs1 != fs2:
+            print("Can only compare signals of same sampling rate")
+            exit(-1)
+
+        cmp_result = audioarray.compare(s1, s2, fs1, per_frame=False, offset_test_start=compare_args.test_start_offset)
+        exit(cmp_result["max_abs_diff"])
+
+    parser_compare = subparsers.add_parser("compare", help="Compare two wav files for bitexactness")
+    parser_compare.add_argument("ref_file")
+    parser_compare.add_argument("test_file")
+    parser_compare.add_argument("test_start_offset", default=0)
+    parser_compare.set_defaults(func=compare_wrapper)
+
     args = parser.parse_args()
     args.func(args)
-- 
GitLab


From f26b93b04f840f64cdc0ddcee7c3b4341c320f6f Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Thu, 6 Jun 2024 16:07:43 +0200
Subject: [PATCH 07/30] set type of offset index

---
 scripts/pyaudio3dtools/audiofile.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/pyaudio3dtools/audiofile.py b/scripts/pyaudio3dtools/audiofile.py
index 405a60d870..cda43d5ef7 100644
--- a/scripts/pyaudio3dtools/audiofile.py
+++ b/scripts/pyaudio3dtools/audiofile.py
@@ -797,7 +797,7 @@ if __name__ == "__main__":
     parser_compare = subparsers.add_parser("compare", help="Compare two wav files for bitexactness")
     parser_compare.add_argument("ref_file")
     parser_compare.add_argument("test_file")
-    parser_compare.add_argument("test_start_offset", default=0)
+    parser_compare.add_argument("test_start_offset", default=0, type=int)
     parser_compare.set_defaults(func=compare_wrapper)
 
     args = parser.parse_args()
-- 
GitLab


From 9f1c660786c33f6fc8a8c6e76650c1a047f9553f Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Thu, 6 Jun 2024 16:11:24 +0200
Subject: [PATCH 08/30] convert offset in ms to samples for slicing

---
 scripts/pyaudio3dtools/audioarray.py | 5 +++--
 scripts/pyaudio3dtools/audiofile.py  | 4 ++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/scripts/pyaudio3dtools/audioarray.py b/scripts/pyaudio3dtools/audioarray.py
index 0da5238ca2..4c06ace848 100644
--- a/scripts/pyaudio3dtools/audioarray.py
+++ b/scripts/pyaudio3dtools/audioarray.py
@@ -232,7 +232,7 @@ def compare(
     fs: int,
     per_frame: bool = True,
     get_mld: bool = False,
-    offset_test_start: int = 0,
+    test_start_offset_ms: int = 0,
 ) -> dict:
     """Compare two audio arrays
 
@@ -255,7 +255,8 @@ def compare(
         Comparison results
     """
     framesize = fs // 50
-    diff = abs(test[offset_test_start:, :] - ref)
+    test_start_offset_samples = int(fs * test_start_offset_ms / 1000)
+    diff = abs(test[test_start_offset_samples:, :] - ref)
     max_diff = int(diff.max())
     result = {
         "bitexact": True,
diff --git a/scripts/pyaudio3dtools/audiofile.py b/scripts/pyaudio3dtools/audiofile.py
index cda43d5ef7..bbee88ca9c 100644
--- a/scripts/pyaudio3dtools/audiofile.py
+++ b/scripts/pyaudio3dtools/audiofile.py
@@ -791,13 +791,13 @@ if __name__ == "__main__":
             print("Can only compare signals of same sampling rate")
             exit(-1)
 
-        cmp_result = audioarray.compare(s1, s2, fs1, per_frame=False, offset_test_start=compare_args.test_start_offset)
+        cmp_result = audioarray.compare(s1, s2, fs1, per_frame=False, test_start_offset_ms=compare_args.test_start_offset_ms)
         exit(cmp_result["max_abs_diff"])
 
     parser_compare = subparsers.add_parser("compare", help="Compare two wav files for bitexactness")
     parser_compare.add_argument("ref_file")
     parser_compare.add_argument("test_file")
-    parser_compare.add_argument("test_start_offset", default=0, type=int)
+    parser_compare.add_argument("test_start_offset_ms", default=0, type=int)
     parser_compare.set_defaults(func=compare_wrapper)
 
     args = parser.parse_args()
-- 
GitLab


From 166d192ffd02b54376453a267fc25b52eea905a3 Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Thu, 6 Jun 2024 16:24:30 +0200
Subject: [PATCH 09/30] add intentional non-BE for voip to test detection

---
 apps/decoder.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/apps/decoder.c b/apps/decoder.c
index 2cd7608cbc..86bbb56f5b 100644
--- a/apps/decoder.c
+++ b/apps/decoder.c
@@ -3578,6 +3578,10 @@ static ivas_error decodeVoIP(
 
     if ( nSamplesFlushed )
     {
+        for (int i = 0; i < 10; i++)
+        {
+            pcmBuf[i] += 1;
+        }
         /* Write current frame */
         if ( ( error = AudioFileWriter_write( afWriter, pcmBuf, nSamplesFlushed * nOutChannels ) ) != IVAS_ERR_OK )
         {
-- 
GitLab


From 1fb8c0aeb04f7e7cd6e64f759897094306f643c6 Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Thu, 6 Jun 2024 16:28:39 +0200
Subject: [PATCH 10/30] try withpout parallel execution

---
 ci/ivas_voip_be_test.sh | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/ci/ivas_voip_be_test.sh b/ci/ivas_voip_be_test.sh
index 5a2e7f9beb..c5bba88357 100755
--- a/ci/ivas_voip_be_test.sh
+++ b/ci/ivas_voip_be_test.sh
@@ -123,13 +123,14 @@ for ref in "$output_dir_default_dec"/*.wav; do
 
     # Print paths of compared files, since the script doesn't do it
     printf "\nComparing %s and %s\n" "$ref" "$cut" | tee -a voip_be_test_output.txt
-    $python_audiofile_script_path compare "$ref" "$cut" "$jbm_offset" &
-    pids="$pids $!"
+    $python_audiofile_script_path compare "$ref" "$cut" "$jbm_offset"
+    echo "$?t"
+    # pids="$pids $!"
 done
 
-for pid in $pids; do
-    wait $pid || let "all_be=0"
-done
+# for pid in $pids; do
+#     wait $pid || let "all_be=0"
+# done
 
 if [ $all_be -eq 1 ]; then
     printf "\n\nAll tested conditions are bit-exact\n" | tee -a voip_be_test_output.txt
-- 
GitLab


From 671b54a55400eeca0de425d88914f914f444ca66 Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Thu, 6 Jun 2024 16:29:54 +0200
Subject: [PATCH 11/30] fix errors in loop

---
 ci/ivas_voip_be_test.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/ivas_voip_be_test.sh b/ci/ivas_voip_be_test.sh
index c5bba88357..008927bb31 100755
--- a/ci/ivas_voip_be_test.sh
+++ b/ci/ivas_voip_be_test.sh
@@ -118,12 +118,12 @@ all_be=1
 pids=""
 
 for ref in "$output_dir_default_dec"/*.wav; do
-    cut=${ref/$output_dir_default_dec/$output_dir_voip}
+    cut=${ref/$output_dir_default/$output_dir_voip}
     cut=${cut/".dec."/"_jbm_dly_error_profile_0_dat.dec."}
 
     # Print paths of compared files, since the script doesn't do it
     printf "\nComparing %s and %s\n" "$ref" "$cut" | tee -a voip_be_test_output.txt
-    $python_audiofile_script_path compare "$ref" "$cut" "$jbm_offset"
+    python3 $python_audiofile_script_path compare "$ref" "$cut" "$jbm_offset"
     echo "$?t"
     # pids="$pids $!"
 done
-- 
GitLab


From 45bf0aff7518f8ebc95326e76fdc31f7cd041b92 Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Fri, 7 Jun 2024 09:55:32 +0200
Subject: [PATCH 12/30] use batch_comp_audio for comparison

---
 ci/ivas_voip_be_test.sh     | 23 +++++------------------
 scripts/batch_comp_audio.py | 29 +++++++++++++++++++++++------
 2 files changed, 28 insertions(+), 24 deletions(-)

diff --git a/ci/ivas_voip_be_test.sh b/ci/ivas_voip_be_test.sh
index 008927bb31..9e0549a0c1 100755
--- a/ci/ivas_voip_be_test.sh
+++ b/ci/ivas_voip_be_test.sh
@@ -113,28 +113,15 @@ python_audiofile_script_path=$python_audio_module_path/pyaudio3dtools/audiofile.
 
 jbm_offset=60
 output_dir_default_dec="$output_dir_default"/dec
+output_dir_jbm = ${ref/$output_dir_default/$output_dir_voip}
 
-all_be=1
-pids=""
+num_diff=0
+num_diff=$(python3 $python_audio_module_path/batch_comp_audio.py $output_dir_default_dec $output_dir_jbm -ds --tool pyaudio3dtools --test_offset_ms $jbm_offset)
 
-for ref in "$output_dir_default_dec"/*.wav; do
-    cut=${ref/$output_dir_default/$output_dir_voip}
-    cut=${cut/".dec."/"_jbm_dly_error_profile_0_dat.dec."}
 
-    # Print paths of compared files, since the script doesn't do it
-    printf "\nComparing %s and %s\n" "$ref" "$cut" | tee -a voip_be_test_output.txt
-    python3 $python_audiofile_script_path compare "$ref" "$cut" "$jbm_offset"
-    echo "$?t"
-    # pids="$pids $!"
-done
-
-# for pid in $pids; do
-#     wait $pid || let "all_be=0"
-# done
-
-if [ $all_be -eq 1 ]; then
+if [ $num_diff == 0 ]; then
     printf "\n\nAll tested conditions are bit-exact\n" | tee -a voip_be_test_output.txt
 else
-    printf "\n\nBitexactness problems found!\n" | tee -a voip_be_test_output.txt
+    =rintf "\n\nBitexactness problems found!\n" | tee -a voip_be_test_output.txt
     exit 1;
 fi
diff --git a/scripts/batch_comp_audio.py b/scripts/batch_comp_audio.py
index cfc546f222..56a6b7d502 100755
--- a/scripts/batch_comp_audio.py
+++ b/scripts/batch_comp_audio.py
@@ -49,16 +49,21 @@ SEG_SNR_EXPR = r"Seg. SNR\s+=(.+)dB"
 DIFF_EXPR = r"Max Diff\s+=\s+(\d+)"
 DIFF_STR = {
     "CompAudio": "{label} Max. diff (PCM) for file {f}: {diff}",
+    "pyaudio3dtools": "{label} Max. diff (PCM) for file {f}: {diff}",
     "mld": "{label} MLD diff for file {f}: {diff}",
 }
 
 
 def main(args):
     tool = args.tool
-    if shutil.which(tool) is None:
+    if tool != "pyaudio3dtools" and shutil.which(tool) is None:
         print(f"{tool} not in PATH - abort.")
         sys.exit(-1)
 
+    test_offset_ms = args.test_offset_ms
+    if tool != "pyaudio3dtools":
+        test_offset_ms = 0
+
     num_files_diff = 0
 
     with OutFileManager(args.out_file) as out_file:
@@ -77,7 +82,7 @@ def main(args):
             # if only one thread is passed, do everything in the main thread
             # to allow for meaningful debugging if needed
             for f in common_files:
-                compare_files(f, fol1, fol2, outputs, tool)
+                compare_files(f, fol1, fol2, outputs, tool, test_offset_ms)
         else:
             with concurrent.futures.ThreadPoolExecutor(
                 max_workers=args.num_threads
@@ -89,6 +94,7 @@ def main(args):
                     repeat(fol2),
                     repeat(outputs),
                     repeat(tool),
+                    repeat(test_offset_ms)
                 )
 
         if args.sort:
@@ -98,7 +104,7 @@ def main(args):
 
         # write csv header
         if out_file is not None:
-            if tool == "CompAudio":
+            if tool == "CompAudio" or tool == "pyaudio3dtools":
                 out_file.write("filename,diff\n")
             elif tool == "mld":
                 out_file.write("filename,mld\n")
@@ -106,7 +112,7 @@ def main(args):
         for f, tool_output in out.items():
             if tool == "CompAudio":
                 diff, snr, gain, seg_snr = tool_output
-            elif tool == "mld":
+            elif tool == "mld" or tool == "pyaudio3dtools":
                 diff = tool_output
 
             if diff > 0:
@@ -136,7 +142,7 @@ def main(args):
     return num_files_diff
 
 
-def compare_files(f, fol1, fol2, outputs_dict, tool):
+def compare_files(f, fol1, fol2, outputs_dict, tool, test_offset_ms):
     """
     Compare file f in both folders fol1 and fol2 using the given tool and
     store the parsed difference in outputs_dict.
@@ -157,6 +163,11 @@ def compare_files(f, fol1, fol2, outputs_dict, tool):
         s2, fs2 = readfile(f2, outdtype="int16")
         cmp_result = compare(s1, s2, fs1, per_frame=False, get_mld=True)
         tool_output = cmp_result["MLD"]
+    elif tool == "pyaudio3dtools":
+        s1, fs1 = readfile(f1, outdtype="int16")
+        s2, fs2 = readfile(f2, outdtype="int16")
+        cmp_result = compare(s1, s2, fs1, per_frame=False, test_start_offset_ms=test_offset_ms)
+        tool_output = cmp_result["max_abs_diff"]
 
     with threading.Lock():
         outputs_dict.update({f: tool_output})
@@ -286,10 +297,16 @@ if __name__ == "__main__":
     )
     parser.add_argument(
         "--tool",
-        choices=["mld", "CompAudio"],
+        choices=["mld", "CompAudio", "pyaudio3dtools"],
         default="CompAudio",
         help="Compare tool to run",
     )
+    parser.add_argument(
+            "--test_offset_ms",
+            type=int,
+            default=0,
+            help="Offset in miliseconds that is ignored at the start of the files in folder2 (only used if tool=pyaudio3dtools)"
+            )
     args = parser.parse_args()
 
     sys.exit(main(args))
-- 
GitLab


From 2a4e731f33e04a87d450330443f2213912cce9bb Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Fri, 7 Jun 2024 10:35:36 +0200
Subject: [PATCH 13/30] fix errors in bash script

---
 ci/ivas_voip_be_test.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ci/ivas_voip_be_test.sh b/ci/ivas_voip_be_test.sh
index 9e0549a0c1..31054b71cd 100755
--- a/ci/ivas_voip_be_test.sh
+++ b/ci/ivas_voip_be_test.sh
@@ -113,11 +113,11 @@ python_audiofile_script_path=$python_audio_module_path/pyaudio3dtools/audiofile.
 
 jbm_offset=60
 output_dir_default_dec="$output_dir_default"/dec
-output_dir_jbm = ${ref/$output_dir_default/$output_dir_voip}
+output_dir_jbm_dec = ${output_dir_default_dec/$output_dir_default/$output_dir_voip}
 
 num_diff=0
-num_diff=$(python3 $python_audio_module_path/batch_comp_audio.py $output_dir_default_dec $output_dir_jbm -ds --tool pyaudio3dtools --test_offset_ms $jbm_offset)
-
+(python3 $python_audio_module_path/batch_comp_audio.py $output_dir_default_dec $output_dir_jbm -ds --tool pyaudio3dtools --test_offset_ms $jbm_offset
+num_diff=$?
 
 if [ $num_diff == 0 ]; then
     printf "\n\nAll tested conditions are bit-exact\n" | tee -a voip_be_test_output.txt
-- 
GitLab


From 853be4c3ef838d91d82a15317eeac8c3d39b9f99 Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Fri, 7 Jun 2024 10:46:12 +0200
Subject: [PATCH 14/30] strip filenames for batch_comp_audio

---
 ci/ivas_voip_be_test.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/ci/ivas_voip_be_test.sh b/ci/ivas_voip_be_test.sh
index 31054b71cd..6b763b6824 100755
--- a/ci/ivas_voip_be_test.sh
+++ b/ci/ivas_voip_be_test.sh
@@ -115,6 +115,11 @@ jbm_offset=60
 output_dir_default_dec="$output_dir_default"/dec
 output_dir_jbm_dec = ${output_dir_default_dec/$output_dir_default/$output_dir_voip}
 
+# strip part of filenames
+for f in output_dir_jbm_dec/*.wav; do
+    mv f ${f/_jbm_dly_error_profile_0_dat/}
+done
+
 num_diff=0
 (python3 $python_audio_module_path/batch_comp_audio.py $output_dir_default_dec $output_dir_jbm -ds --tool pyaudio3dtools --test_offset_ms $jbm_offset
 num_diff=$?
-- 
GitLab


From a392b2c6d80890a68fea01a83a0f19c8d025f7a0 Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Fri, 7 Jun 2024 10:59:44 +0200
Subject: [PATCH 15/30] fix typos and other mistakes

---
 ci/ivas_voip_be_test.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ci/ivas_voip_be_test.sh b/ci/ivas_voip_be_test.sh
index 6b763b6824..c03a713957 100755
--- a/ci/ivas_voip_be_test.sh
+++ b/ci/ivas_voip_be_test.sh
@@ -113,20 +113,20 @@ python_audiofile_script_path=$python_audio_module_path/pyaudio3dtools/audiofile.
 
 jbm_offset=60
 output_dir_default_dec="$output_dir_default"/dec
-output_dir_jbm_dec = ${output_dir_default_dec/$output_dir_default/$output_dir_voip}
+output_dir_jbm_dec=${output_dir_default_dec/$output_dir_default/$output_dir_voip}
 
 # strip part of filenames
-for f in output_dir_jbm_dec/*.wav; do
+for f in $output_dir_jbm_dec/*.wav; do
     mv f ${f/_jbm_dly_error_profile_0_dat/}
 done
 
 num_diff=0
-(python3 $python_audio_module_path/batch_comp_audio.py $output_dir_default_dec $output_dir_jbm -ds --tool pyaudio3dtools --test_offset_ms $jbm_offset
+python3 $python_audio_module_path/batch_comp_audio.py $output_dir_default_dec $output_dir_jbm -ds --tool pyaudio3dtools --test_offset_ms $jbm_offset
 num_diff=$?
 
 if [ $num_diff == 0 ]; then
     printf "\n\nAll tested conditions are bit-exact\n" | tee -a voip_be_test_output.txt
 else
-    =rintf "\n\nBitexactness problems found!\n" | tee -a voip_be_test_output.txt
+    printf "\n\nBitexactness problems found!\n" | tee -a voip_be_test_output.txt
     exit 1;
 fi
-- 
GitLab


From 91ad6000a6dcb2365c70ca5bf3c0d4d85ba4819c Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Fri, 7 Jun 2024 11:14:22 +0200
Subject: [PATCH 16/30] fix more typos

---
 ci/ivas_voip_be_test.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/ivas_voip_be_test.sh b/ci/ivas_voip_be_test.sh
index c03a713957..9ec6477860 100755
--- a/ci/ivas_voip_be_test.sh
+++ b/ci/ivas_voip_be_test.sh
@@ -117,11 +117,11 @@ output_dir_jbm_dec=${output_dir_default_dec/$output_dir_default/$output_dir_voip
 
 # strip part of filenames
 for f in $output_dir_jbm_dec/*.wav; do
-    mv f ${f/_jbm_dly_error_profile_0_dat/}
+    mv $f ${f/_jbm_dly_error_profile_0_dat/}
 done
 
 num_diff=0
-python3 $python_audio_module_path/batch_comp_audio.py $output_dir_default_dec $output_dir_jbm -ds --tool pyaudio3dtools --test_offset_ms $jbm_offset
+python3 $python_audio_module_path/batch_comp_audio.py $output_dir_default_dec $output_dir_jbm_dec -ds --tool pyaudio3dtools --test_offset_ms $jbm_offset
 num_diff=$?
 
 if [ $num_diff == 0 ]; then
-- 
GitLab


From 39369f07e2a3514bfc7370663a3cdb67e8c8b78a Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Fri, 7 Jun 2024 11:15:17 +0200
Subject: [PATCH 17/30] Revert "add intentional non-BE for voip to test
 detection"

This reverts commit 166d192ffd02b54376453a267fc25b52eea905a3.
---
 apps/decoder.c | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/apps/decoder.c b/apps/decoder.c
index 86bbb56f5b..2cd7608cbc 100644
--- a/apps/decoder.c
+++ b/apps/decoder.c
@@ -3578,10 +3578,6 @@ static ivas_error decodeVoIP(
 
     if ( nSamplesFlushed )
     {
-        for (int i = 0; i < 10; i++)
-        {
-            pcmBuf[i] += 1;
-        }
         /* Write current frame */
         if ( ( error = AudioFileWriter_write( afWriter, pcmBuf, nSamplesFlushed * nOutChannels ) ) != IVAS_ERR_OK )
         {
-- 
GitLab


From 8a6dfd5ce523f2c31e92edadbe85b1ec261ca131 Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Mon, 10 Jun 2024 16:36:27 +0200
Subject: [PATCH 18/30] first version of jbm be test port to pytest

---
 tests/conftest.py                            | 37 +++++++++-
 tests/constants.py                           |  6 ++
 tests/test_be_for_jbm_neutral_dly_profile.py | 74 ++++++++++++++++++++
 3 files changed, 115 insertions(+), 2 deletions(-)
 create mode 100644 tests/test_be_for_jbm_neutral_dly_profile.py

diff --git a/tests/conftest.py b/tests/conftest.py
index cec06c2061..6ca14ee056 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -487,6 +487,7 @@ class DecoderFrontend:
         output_path: Path,
         quiet_mode: Optional[bool] = True,
         plc_file: Optional[Path] = None,
+        netsim_profile: Optional[Path] = None,
         add_option_list: Optional[list] = None,
         run_dir: Optional[Path] = None,
     ) -> None:
@@ -525,11 +526,43 @@ class DecoderFrontend:
             try:
                 if not os.path.exists(str(input_bitstream_path) + eid_output_suffix):
                     result = run(eid_command, check=True, cwd=run_dir)
-            except Exception:
-                pytest.fail("eid-xor operation failed!")
+            except Exception as e:
+                pytest.fail(f"eid-xor operation failed! - {e}")
 
             input_bitstream_path += eid_output_suffix
 
+        if netsim_profile is not None:
+            system = platform.system()
+
+            # TODO: centralize this in a utils file
+            if system == "Windows":
+                netsim_path = "./scripts/tools/Win32/networkSimulator_g192.exe"
+            elif system == "Linux":
+                netsim_path = "./scripts/tools/Linux/networkSimulator_g192"
+            elif system == "Darwin":
+                netsim_path = "./scripts/tools/Darwin/networkSimulator_g192"
+            else:
+                raise ValueError(f'Wrong system "{system}"!')
+
+            if not os.path.isfile(netsim_path):
+                raise FileNotFoundError(f"network simulator binary {netsim_path} not found!\n")
+            netsim_bitstream_path = input_bitstream_path.with_suffix(".netsimout")
+            tracefile_path = input_bitstream_path.with_suffix(".netsimtrace")
+            # TODO: need to check if the "1" works with every profile
+            netsim_command = [netsim_path, netsim_profile, input_bitstream_path, netsim_bitstream_path, tracefile_path, "1"]
+            print(netsim_command)
+            try:
+                run(netsim_command, check=True, cwd=run_dir)
+            except Exception as e:
+                pytest.fail(f"netsim operation failed! - {e}")
+
+            input_bitstream_path = netsim_bitstream_path
+            voip_opt = ["-voip"]
+            if add_option_list is None:
+                add_option_list = voip_opt
+            elif "-voip" not in add_option_list:
+                add_option_list.extend(voip_opt)
+
         if add_option_list is not None:
             command.extend(add_option_list)
 
diff --git a/tests/constants.py b/tests/constants.py
index 453c07dca8..b00eca6d1b 100644
--- a/tests/constants.py
+++ b/tests/constants.py
@@ -1,3 +1,9 @@
+import pathlib
+
+HERE = pathlib.Path(__file__).parent.absolute()
+SCRIPTS_DIR = HERE.parent.joinpath("scripts")
+TESTV_DIR = SCRIPTS_DIR.joinpath("testv")
+
 # regex patterns for parsing the output from cmp_pcm -> mainly for BASOP ci
 MLD_PATTERN = r"MLD: ([\d\.]*)"
 MAX_DIFF_PATTERN = r"MAXIMUM ABS DIFF: (\d*)"
diff --git a/tests/test_be_for_jbm_neutral_dly_profile.py b/tests/test_be_for_jbm_neutral_dly_profile.py
new file mode 100644
index 0000000000..1a14f21561
--- /dev/null
+++ b/tests/test_be_for_jbm_neutral_dly_profile.py
@@ -0,0 +1,74 @@
+import pytest
+import pathlib
+import sys
+from tempfile import TemporaryDirectory
+
+from .constants import TESTV_DIR, SCRIPTS_DIR
+
+sys.path.append(SCRIPTS_DIR)
+from pyaudio3dtools import audiofile, audioarray
+
+
+TESTCASES = [
+    ["stereo", 32000, "EXT"],
+    ["stereo", 48000, "EXT"],
+]
+DLY_PROFILE = SCRIPTS_DIR.joinpath("dly_error_profiles/dly_error_profile_0.dat")
+
+
+def get_options(in_format):
+    options = list()
+    if in_format == "stereo":
+        options.append("-stereo")
+
+    return options
+
+
+@pytest.mark.parametrize("in_format,bitrate,out_format", TESTCASES)
+def test_be_for_jbm_neutral_dly_profile(
+    in_format, bitrate, out_format, dut_encoder_frontend, dut_decoder_frontend
+):
+    with TemporaryDirectory() as tmp_dir:
+        tmp_dir = pathlib.Path(tmp_dir)
+
+        # run encoder
+        bitstream_file = tmp_dir.joinpath(f"{in_format}.192").absolute()
+        sampling_rate_khz = 48
+        inp_file = "stvST48c.wav"
+        input_file = TESTV_DIR.joinpath(inp_file)
+        options = get_options(in_format)
+        dut_encoder_frontend.run(
+            bitrate,
+            sampling_rate_khz,
+            input_file,
+            bitstream_file,
+            add_option_list=options,
+            run_dir=tmp_dir,
+        )
+
+        # run decoder without network simulation
+        output = tmp_dir.joinpath(f"{in_format}-{bitrate}-{out_format}.wav").absolute()
+        dut_decoder_frontend.run(out_format, sampling_rate_khz, bitstream_file, output)
+
+        # run decoder with network simulation
+        output_jbm = output.with_suffix(".jbm-0.wav")
+        dut_decoder_frontend.run(
+            out_format,
+            sampling_rate_khz,
+            bitstream_file,
+            output_jbm,
+            netsim_profile=DLY_PROFILE,
+        )
+
+        # compare no-jbm and jbm output
+        x, _ = audiofile.readfile(output)
+        x_jbm, _ = audiofile.readfile(output_jbm)
+        cmp_result = audioarray.compare(
+            x,
+            x_jbm,
+            fs=sampling_rate_khz * 1000,
+            per_frame=False,
+            test_start_offset_ms=60,
+        )
+        if not cmp_result["bitexact"]:
+            pytest.fail("Difference between no jbm and zero-delay jbm decoding!")
-- 
GitLab


From 23f4f78d773b7236f10d739cd7186743c9ddcb0a Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Tue, 11 Jun 2024 09:39:03 +0200
Subject: [PATCH 19/30] add more modes

---
 tests/test_be_for_jbm_neutral_dly_profile.py | 92 ++++++++++++++++++--
 1 file changed, 85 insertions(+), 7 deletions(-)

diff --git a/tests/test_be_for_jbm_neutral_dly_profile.py b/tests/test_be_for_jbm_neutral_dly_profile.py
index 1a14f21561..437838f241 100644
--- a/tests/test_be_for_jbm_neutral_dly_profile.py
+++ b/tests/test_be_for_jbm_neutral_dly_profile.py
@@ -1,6 +1,7 @@
 import pytest
 import pathlib
 import sys
+import re
 from tempfile import TemporaryDirectory
 
 from .constants import TESTV_DIR, SCRIPTS_DIR
@@ -11,19 +12,94 @@ from pyaudio3dtools import audiofile, audioarray
 
 TESTCASES = [
     ["stereo", 32000, "EXT"],
-    ["stereo", 48000, "EXT"],
+    ["stereo", 48000, "7_1_4"],
+    ["ISM4", 48000, "BINAURAL"],
+    ["ISM3", 64000, "BINAURAL"],
+    ["MASA1TC", 24400, "BINAURAL"],
+    ["MASA2TC", 80000, "BINAURAL"],
+    ["MC_5_1", 128000, "BINAURAL"],
+    ["MC_5_1_4", 48000, "BINAURAL"],
+    ["MC_7_1", 96000, "BINAURAL"],
+    ["MC_7_1_4", 160000, "7_1_4"],
+    ["HOA3", 64000, "BINAURAL"],
+    ["FOA", 256000, "BINAURAL"],
+    ["OMASA_ISM1", 512000, "BINAURAL"],
+    ["OMASA_ISM2", 24400, "MONO"],
+    ["OMASA_ISM3", 80000, "7_1_4"],
+    ["OMASA_ISM4", 64000, "HOA3"],
+    ["OSBA_ISM2_HOA2", 64000, "BINAURAL"],
+    ["OSBA_ISM4_HOA2", 512000, "BINAURAL"],
 ]
 DLY_PROFILE = SCRIPTS_DIR.joinpath("dly_error_profiles/dly_error_profile_0.dat")
 
 
-def get_options(in_format):
+def get_options(in_format, bitrate):
     options = list()
+
+    SBA_FORMATS = ["FOA", "HOA2", "HOA3"]
     if in_format == "stereo":
+        options.append("-dtx")
         options.append("-stereo")
+    elif (match := re.fullmatch(r"ISM(\d)", in_format)) is not None:
+        n_ism = int(match.groups()[0])
+        options.append("-dtx")
+        options.extend(f"-ism {n_ism}".split())
+        for i in range(1, n_ism + 1):
+            options.append(str(TESTV_DIR.joinpath(f"stvISM{i}.csv")))
+    elif (match := re.fullmatch(r"MASA(\d)TC", in_format)) is not None:
+        n_tcs = int(match.groups()[0])
+        options.append("-dtx")
+        options.extend(f"-masa {n_tcs}".split())
+        options.append(str(TESTV_DIR.joinpath(f"stv2MASA{n_tcs}TC48c.met")))
+    elif (match := re.fullmatch(r"MC_(.*)", in_format)) is not None:
+        mc_format = match.groups()[0]
+        options.extend(f"-mc {mc_format}".split())
+    elif in_format in SBA_FORMATS:
+        if bitrate <= 80000:
+            options.append("-dtx")
+        options.extend(f"-sba {SBA_FORMATS.index(in_format) + 1}".split())
+    elif (match := re.fullmatch(r"OMASA_ISM(\d)", in_format)) is not None:
+        n_ism = int(match.groups()[0])
+        n_tcs = 1
+        options.extend(f"-ism_masa {n_ism} {n_tcs}".split())
+        for i in range(1, n_ism + 1):
+            options.append(str(TESTV_DIR.joinpath(f"stvISM{i}.csv")))
+        options.append(
+            str(TESTV_DIR.joinpath(f"stvOMASA_{n_ism}ISM_1MASA{n_tcs}TC48c.met"))
+        )
+    elif (match := re.fullmatch(r"OSBA_ISM(\d)_(.*)", in_format)) is not None:
+        n_ism = int(match.groups()[0])
+        # NOTE: this will break if FOA is used in the future
+        n_sba = int(match.groups()[1][-1])
+        options.extend(f"-ism_sba {n_ism}, -{n_sba}".split())
+        for i in range(1, n_ism + 1):
+            options.append(str(TESTV_DIR.joinpath(f"stvISM{i}.csv")))
 
     return options
 
 
+INPUT_FILES = {
+    "stereo": "stvST48n.wav",
+    "ISM3": "stv3ISM48s.wav",
+    "ISM4": "stv4ISM48n.wav",
+    "MASA1TC": "stv2MASA1TC48c.wav",
+    "MASA2TC": "stv2MASA2TC48c.wav",
+    "MC_5_1": "stv51MC48c.wav",
+    "MC_5_1_4": "stv514MC48c.wav",
+    "MC_7_1": "stv71MC48c.wav",
+    "MC_7_1_4": "stv714MC48c.wav",
+    "FOA": "stvFOA48c.wav",
+    "HOA2": "stv2OA48c.wav",
+    "HOA3": "stv3OA48c.wav",
+    "OMASA_ISM1": "stvOMASA_1ISM_2MASA1TC48c.wav",
+    "OMASA_ISM2": "stvOMASA_2ISM_2MASA1TC48c.wav",
+    "OMASA_ISM3": "stvOMASA_3ISM_2MASA1TC48c.wav",
+    "OMASA_ISM4": "stvOMASA_24SM_2MASA1TC48c.wav",
+    "OSBA_ISM2_HOA2": "stvOSBA_2ISM_2OA48c.wav",
+    "OSBA_ISM4_HOA2": "stvOSBA_4ISM_2OA48c.wav",
+}
+
+
 @pytest.mark.parametrize("in_format,bitrate,out_format", TESTCASES)
 def test_be_for_jbm_neutral_dly_profile(
     in_format, bitrate, out_format, dut_encoder_frontend, dut_decoder_frontend
@@ -34,9 +110,8 @@ def test_be_for_jbm_neutral_dly_profile(
         # run encoder
         bitstream_file = tmp_dir.joinpath(f"{in_format}.192").absolute()
         sampling_rate_khz = 48
-        inp_file = "stvST48c.wav"
-        input_file = TESTV_DIR.joinpath(inp_file)
-        options = get_options(in_format)
+        input_file = TESTV_DIR.joinpath(INPUT_FILES[in_format])
+        options = get_options(in_format, bitrate)
         dut_encoder_frontend.run(
             bitrate,
             sampling_rate_khz,
@@ -63,12 +138,15 @@ def test_be_for_jbm_neutral_dly_profile(
         # compare no-jbm and jbm output
         x, _ = audiofile.readfile(output)
         x_jbm, _ = audiofile.readfile(output_jbm)
+        delay = x_jbm.shape[0] - x.shape[0]
+        print(delay)
+        print(x_jbm[delay:, :])
+        print(x_jbm[delay:, :].shape)
         cmp_result = audioarray.compare(
             x,
-            x_jbm,
+            x_jbm[delay:, :],
             fs=sampling_rate_khz * 1000,
             per_frame=False,
-            test_start_offset_ms=60,
         )
         if not cmp_result["bitexact"]:
             pytest.fail("Difference between no jbm and zero-delay jbm decoding!")
-- 
GitLab


From c414ddeb0743b5084091a977e098315458f4e13f Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Tue, 11 Jun 2024 09:46:58 +0200
Subject: [PATCH 20/30] use pytest implementation in CI

---
 .gitlab-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a3e86272ea..0a45807447 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1058,11 +1058,11 @@ voip-be-on-merge-request:
     - .test-job-linux-needs-testv-dir
     - .rules-merge-request
   stage: test
-  needs: ["build-codec-linux-make", "codec-smoke-test"]
+  needs: ["build-codec-linux-make"]
   timeout: "10 minutes"
   script:
     - *print-common-info
-    - bash ci/ivas_voip_be_test.sh
+    - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py
 
 clang-format-check:
   extends:
-- 
GitLab


From bf3e381ad94d7bd797644b6c94dd7fd6e5e207e0 Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Tue, 11 Jun 2024 09:51:13 +0200
Subject: [PATCH 21/30] build codec in voip be test

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index efa6e3992a..ee73c322d2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1016,6 +1016,7 @@ voip-be-on-merge-request:
   timeout: "10 minutes"
   script:
     - *print-common-info
+    - make -j
     - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py
 
 clang-format-check:
-- 
GitLab


From 5579f78312a64efee02379e5da5855ec2df4b323 Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Tue, 11 Jun 2024 10:09:48 +0200
Subject: [PATCH 22/30] fix Omasa cases

---
 tests/test_be_for_jbm_neutral_dly_profile.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/test_be_for_jbm_neutral_dly_profile.py b/tests/test_be_for_jbm_neutral_dly_profile.py
index 437838f241..e059d2bf40 100644
--- a/tests/test_be_for_jbm_neutral_dly_profile.py
+++ b/tests/test_be_for_jbm_neutral_dly_profile.py
@@ -60,7 +60,7 @@ def get_options(in_format, bitrate):
         options.extend(f"-sba {SBA_FORMATS.index(in_format) + 1}".split())
     elif (match := re.fullmatch(r"OMASA_ISM(\d)", in_format)) is not None:
         n_ism = int(match.groups()[0])
-        n_tcs = 1
+        n_tcs = 2
         options.extend(f"-ism_masa {n_ism} {n_tcs}".split())
         for i in range(1, n_ism + 1):
             options.append(str(TESTV_DIR.joinpath(f"stvISM{i}.csv")))
@@ -91,10 +91,10 @@ INPUT_FILES = {
     "FOA": "stvFOA48c.wav",
     "HOA2": "stv2OA48c.wav",
     "HOA3": "stv3OA48c.wav",
-    "OMASA_ISM1": "stvOMASA_1ISM_2MASA1TC48c.wav",
-    "OMASA_ISM2": "stvOMASA_2ISM_2MASA1TC48c.wav",
-    "OMASA_ISM3": "stvOMASA_3ISM_2MASA1TC48c.wav",
-    "OMASA_ISM4": "stvOMASA_24SM_2MASA1TC48c.wav",
+    "OMASA_ISM1": "stvOMASA_1ISM_2MASA2TC48c.wav",
+    "OMASA_ISM2": "stvOMASA_2ISM_2MASA2TC48c.wav",
+    "OMASA_ISM3": "stvOMASA_3ISM_2MASA2TC48c.wav",
+    "OMASA_ISM4": "stvOMASA_4ISM_2MASA2TC48c.wav",
     "OSBA_ISM2_HOA2": "stvOSBA_2ISM_2OA48c.wav",
     "OSBA_ISM4_HOA2": "stvOSBA_4ISM_2OA48c.wav",
 }
-- 
GitLab


From 7667a252fac3cb4b465f05031a3a0dcbb4928592 Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Tue, 11 Jun 2024 13:29:23 +0200
Subject: [PATCH 23/30] change length check

---
 tests/test_be_for_jbm_neutral_dly_profile.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/tests/test_be_for_jbm_neutral_dly_profile.py b/tests/test_be_for_jbm_neutral_dly_profile.py
index e059d2bf40..e51bd8bca7 100644
--- a/tests/test_be_for_jbm_neutral_dly_profile.py
+++ b/tests/test_be_for_jbm_neutral_dly_profile.py
@@ -31,7 +31,7 @@ TESTCASES = [
     ["OSBA_ISM4_HOA2", 512000, "BINAURAL"],
 ]
 DLY_PROFILE = SCRIPTS_DIR.joinpath("dly_error_profiles/dly_error_profile_0.dat")
-
+JBM_NEUTRAL_DELAY_MS = 60
 
 def get_options(in_format, bitrate):
     options = list()
@@ -138,13 +138,17 @@ def test_be_for_jbm_neutral_dly_profile(
         # compare no-jbm and jbm output
         x, _ = audiofile.readfile(output)
         x_jbm, _ = audiofile.readfile(output_jbm)
-        delay = x_jbm.shape[0] - x.shape[0]
-        print(delay)
-        print(x_jbm[delay:, :])
-        print(x_jbm[delay:, :].shape)
+        
+        # strip jbm delay
+        delay_samples = JBM_NEUTRAL_DELAY_MS * sampling_rate_khz
+        x_jbm = x_jbm[delay_samples:, :]
+        # TODO: if #1122 results in the JBM output always being longer,
+        # then this should be removed and a length check with failure otherwise added
+        min_len = min(len(x), len(x_jbm))
+
         cmp_result = audioarray.compare(
-            x,
-            x_jbm[delay:, :],
+            x[:min_len, :],
+            x_jbm[:min_len, :],
             fs=sampling_rate_khz * 1000,
             per_frame=False,
         )
-- 
GitLab


From dda207e0ac26dbf855552245ab4046ba9c10a080 Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Fri, 21 Jun 2024 14:19:10 +0200
Subject: [PATCH 24/30] no DTX for now

---
 tests/test_be_for_jbm_neutral_dly_profile.py | 61 ++++++++++----------
 1 file changed, 32 insertions(+), 29 deletions(-)

diff --git a/tests/test_be_for_jbm_neutral_dly_profile.py b/tests/test_be_for_jbm_neutral_dly_profile.py
index e51bd8bca7..c4d5ccf427 100644
--- a/tests/test_be_for_jbm_neutral_dly_profile.py
+++ b/tests/test_be_for_jbm_neutral_dly_profile.py
@@ -6,57 +6,60 @@ from tempfile import TemporaryDirectory
 
 from .constants import TESTV_DIR, SCRIPTS_DIR
 
-sys.path.append(SCRIPTS_DIR)
+sys.path.append(str(SCRIPTS_DIR))
 from pyaudio3dtools import audiofile, audioarray
 
 
+DTX_ON = "DTX_ON"
+DTX_OFF = "DTX_OFF"
+
+
 TESTCASES = [
-    ["stereo", 32000, "EXT"],
-    ["stereo", 48000, "7_1_4"],
-    ["ISM4", 48000, "BINAURAL"],
-    ["ISM3", 64000, "BINAURAL"],
-    ["MASA1TC", 24400, "BINAURAL"],
-    ["MASA2TC", 80000, "BINAURAL"],
-    ["MC_5_1", 128000, "BINAURAL"],
-    ["MC_5_1_4", 48000, "BINAURAL"],
-    ["MC_7_1", 96000, "BINAURAL"],
-    ["MC_7_1_4", 160000, "7_1_4"],
-    ["HOA3", 64000, "BINAURAL"],
-    ["FOA", 256000, "BINAURAL"],
-    ["OMASA_ISM1", 512000, "BINAURAL"],
-    ["OMASA_ISM2", 24400, "MONO"],
-    ["OMASA_ISM3", 80000, "7_1_4"],
-    ["OMASA_ISM4", 64000, "HOA3"],
-    ["OSBA_ISM2_HOA2", 64000, "BINAURAL"],
-    ["OSBA_ISM4_HOA2", 512000, "BINAURAL"],
+    ["stereo", 32000, "EXT", "DTX_OFF"],
+    ["stereo", 48000, "7_1_4", "DTX_OFF"],
+    ["ISM4", 48000, "BINAURAL", "DTX_OFF"],
+    ["ISM3", 64000, "BINAURAL", "DTX_OFF"],
+    ["MASA1TC", 24400, "BINAURAL", "DTX_OFF"],
+    ["MASA2TC", 80000, "BINAURAL", "DTX_OFF"],
+    ["MC_5_1", 128000, "BINAURAL", "DTX_OFF"],
+    ["MC_5_1_4", 48000, "BINAURAL", "DTX_OFF"],
+    ["MC_7_1", 96000, "BINAURAL", "DTX_OFF"],
+    ["MC_7_1_4", 160000, "7_1_4", "DTX_OFF"],
+    ["HOA3", 64000, "BINAURAL", "DTX_OFF"],
+    ["FOA", 256000, "BINAURAL", "DTX_OFF"],
+    ["OMASA_ISM1", 512000, "BINAURAL", "DTX_OFF"],
+    ["OMASA_ISM2", 24400, "MONO", "DTX_OFF"],
+    ["OMASA_ISM3", 80000, "7_1_4", "DTX_OFF"],
+    ["OMASA_ISM4", 64000, "HOA3", "DTX_OFF"],
+    ["OSBA_ISM2_HOA2", 64000, "BINAURAL", "DTX_OFF"],
+    ["OSBA_ISM4_HOA2", 512000, "BINAURAL", "DTX_OFF"],
 ]
 DLY_PROFILE = SCRIPTS_DIR.joinpath("dly_error_profiles/dly_error_profile_0.dat")
 JBM_NEUTRAL_DELAY_MS = 60
 
-def get_options(in_format, bitrate):
+
+def get_options(in_format, bitrate, dtx):
     options = list()
 
+    if dtx:
+        options.append("-dtx")
+
     SBA_FORMATS = ["FOA", "HOA2", "HOA3"]
     if in_format == "stereo":
-        options.append("-dtx")
         options.append("-stereo")
     elif (match := re.fullmatch(r"ISM(\d)", in_format)) is not None:
         n_ism = int(match.groups()[0])
-        options.append("-dtx")
         options.extend(f"-ism {n_ism}".split())
         for i in range(1, n_ism + 1):
             options.append(str(TESTV_DIR.joinpath(f"stvISM{i}.csv")))
     elif (match := re.fullmatch(r"MASA(\d)TC", in_format)) is not None:
         n_tcs = int(match.groups()[0])
-        options.append("-dtx")
         options.extend(f"-masa {n_tcs}".split())
         options.append(str(TESTV_DIR.joinpath(f"stv2MASA{n_tcs}TC48c.met")))
     elif (match := re.fullmatch(r"MC_(.*)", in_format)) is not None:
         mc_format = match.groups()[0]
         options.extend(f"-mc {mc_format}".split())
     elif in_format in SBA_FORMATS:
-        if bitrate <= 80000:
-            options.append("-dtx")
         options.extend(f"-sba {SBA_FORMATS.index(in_format) + 1}".split())
     elif (match := re.fullmatch(r"OMASA_ISM(\d)", in_format)) is not None:
         n_ism = int(match.groups()[0])
@@ -100,9 +103,9 @@ INPUT_FILES = {
 }
 
 
-@pytest.mark.parametrize("in_format,bitrate,out_format", TESTCASES)
+@pytest.mark.parametrize("in_format,bitrate,out_format,dtx", TESTCASES)
 def test_be_for_jbm_neutral_dly_profile(
-    in_format, bitrate, out_format, dut_encoder_frontend, dut_decoder_frontend
+    in_format, bitrate, out_format, dtx, dut_encoder_frontend, dut_decoder_frontend
 ):
     with TemporaryDirectory() as tmp_dir:
         tmp_dir = pathlib.Path(tmp_dir)
@@ -111,7 +114,7 @@ def test_be_for_jbm_neutral_dly_profile(
         bitstream_file = tmp_dir.joinpath(f"{in_format}.192").absolute()
         sampling_rate_khz = 48
         input_file = TESTV_DIR.joinpath(INPUT_FILES[in_format])
-        options = get_options(in_format, bitrate)
+        options = get_options(in_format, bitrate, dtx == DTX_ON)
         dut_encoder_frontend.run(
             bitrate,
             sampling_rate_khz,
@@ -138,7 +141,7 @@ def test_be_for_jbm_neutral_dly_profile(
         # compare no-jbm and jbm output
         x, _ = audiofile.readfile(output)
         x_jbm, _ = audiofile.readfile(output_jbm)
-        
+
         # strip jbm delay
         delay_samples = JBM_NEUTRAL_DELAY_MS * sampling_rate_khz
         x_jbm = x_jbm[delay_samples:, :]
-- 
GitLab


From a5199c2dfd629d2711c774a03fa439c9f6d41be7 Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Fri, 21 Jun 2024 15:48:52 +0200
Subject: [PATCH 25/30] add more output formats

---
 tests/test_be_for_jbm_neutral_dly_profile.py | 70 ++++++++++++++++----
 1 file changed, 57 insertions(+), 13 deletions(-)

diff --git a/tests/test_be_for_jbm_neutral_dly_profile.py b/tests/test_be_for_jbm_neutral_dly_profile.py
index c4d5ccf427..a135181672 100644
--- a/tests/test_be_for_jbm_neutral_dly_profile.py
+++ b/tests/test_be_for_jbm_neutral_dly_profile.py
@@ -15,24 +15,61 @@ DTX_OFF = "DTX_OFF"
 
 
 TESTCASES = [
+    # stereo
     ["stereo", 32000, "EXT", "DTX_OFF"],
-    ["stereo", 48000, "7_1_4", "DTX_OFF"],
-    ["ISM4", 48000, "BINAURAL", "DTX_OFF"],
+    ["stereo", 48000, "MONO", "DTX_OFF"],
+    ["stereo", 16400, "5_1", "DTX_OFF"],
+    ["stereo", 256000, "7_1_4", "DTX_OFF"],
+    # param ISM
+    ["ISM4", 32000, "BINAURAL_ROOM_REVERB", "DTX_OFF"],
+    ["ISM3", 24400, "STEREO", "DTX_OFF"],
+    ["ISM4", 24400, "5_1_2", "DTX_OFF"],
+    ["ISM3", 32000, "HOA2", "DTX_OFF"],
+    # discrete ISM
     ["ISM3", 64000, "BINAURAL", "DTX_OFF"],
-    ["MASA1TC", 24400, "BINAURAL", "DTX_OFF"],
-    ["MASA2TC", 80000, "BINAURAL", "DTX_OFF"],
-    ["MC_5_1", 128000, "BINAURAL", "DTX_OFF"],
-    ["MC_5_1_4", 48000, "BINAURAL", "DTX_OFF"],
-    ["MC_7_1", 96000, "BINAURAL", "DTX_OFF"],
-    ["MC_7_1_4", 160000, "7_1_4", "DTX_OFF"],
+    ["ISM1", 32000, "BINAURAL_ROOM_IR", "DTX_OFF"],
+    ["ISM2", 96000, "7_1", "DTX_OFF"],
+    ["ISM1", 80000, "MONO", "DTX_OFF"],
+    ["ISM4", 128000, "FOA", "DTX_OFF"],
+    # MASA
+    ["MASA1TC", 24400, "BINAURAL_ROOM_IR", "DTX_OFF"],
+    ["MASA2TC", 80000, "stereo", "DTX_OFF"],
+    ["MASA1TC", 16400, "7_1_4", "DTX_OFF"],
+    ["MASA2TC", 256000, "HOA3", "DTX_OFF"],
+    ["MASA1TC", 128000, "EXT", "DTX_OFF"],
+    # MC
+    # McMasa
+    ["MC_5_1", 16400, "BINAURAL_ROOM_IR", "DTX_OFF"],
+    ["MC_7_1_4", 80000, "mono", "DTX_OFF"],
+    ["MC_5_1_2", 24400, "EXT", "DTX_OFF"],
+    # paramMC
+    ["MC_5_1_2", 48000, "BINAURAL", "DTX_OFF"],
+    ["MC_7_1", 80000, "EXT", "DTX_OFF"],
+    ["MC_7_1_4", 128000, "FOA", "DTX_OFF"],
+    # paramUpmix
+    ["MC_7_1_4", 160000, "stereo", "DTX_OFF"],
+    # discrete MC
+    ["MC_5_1_2", 512000, "BINAURAL_ROOM_REVERB", "DTX_OFF"],
+    ["MC_7_1", 128000, "EXT", "DTX_OFF"],
+    ["MC_7_1_4", 256000, "5_1", "DTX_OFF"],
+    # SBA
     ["HOA3", 64000, "BINAURAL", "DTX_OFF"],
-    ["FOA", 256000, "BINAURAL", "DTX_OFF"],
+    ["FOA", 256000, "BINAURAL_ROOM_REVERB", "DTX_OFF"],
+    ["HOA2", 80000, "EXT", "DTX_OFF"],
+    ["HOA3", 512000, "5_1_4", "DTX_OFF"],
+    ["FOA", 80000, "stereo", "DTX_OFF"],
+    # OMASA
     ["OMASA_ISM1", 512000, "BINAURAL", "DTX_OFF"],
     ["OMASA_ISM2", 24400, "MONO", "DTX_OFF"],
     ["OMASA_ISM3", 80000, "7_1_4", "DTX_OFF"],
     ["OMASA_ISM4", 64000, "HOA3", "DTX_OFF"],
-    ["OSBA_ISM2_HOA2", 64000, "BINAURAL", "DTX_OFF"],
-    ["OSBA_ISM4_HOA2", 512000, "BINAURAL", "DTX_OFF"],
+    ["OMASA_ISM2", 32000, "EXT", "DTX_OFF"],
+    # OSBA
+    ["OSBA_ISM2_HOA2", 64000, "BINAURAL_ROOM_IR", "DTX_OFF"],
+    ["OSBA_ISM4_FOA", 512000, "BINAURAL_ROOM_REVERB", "DTX_OFF"],
+    ["OSBA_ISM3_HOA3", 128000, "EXT", "DTX_OFF"],
+    ["OSBA_ISM2_HOA3", 96000, "5_1", "DTX_OFF"],
+    ["OSBA_ISM1_HOA2", 32000, "mono", "DTX_OFF"],
 ]
 DLY_PROFILE = SCRIPTS_DIR.joinpath("dly_error_profiles/dly_error_profile_0.dat")
 JBM_NEUTRAL_DELAY_MS = 60
@@ -72,8 +109,8 @@ def get_options(in_format, bitrate, dtx):
         )
     elif (match := re.fullmatch(r"OSBA_ISM(\d)_(.*)", in_format)) is not None:
         n_ism = int(match.groups()[0])
-        # NOTE: this will break if FOA is used in the future
-        n_sba = int(match.groups()[1][-1])
+        sba_order_str = match.groups()[1]
+        n_sba = 1 if sba_order_str == "FOA" else str(sba_order_str[-1])
         options.extend(f"-ism_sba {n_ism}, -{n_sba}".split())
         for i in range(1, n_ism + 1):
             options.append(str(TESTV_DIR.joinpath(f"stvISM{i}.csv")))
@@ -83,11 +120,14 @@ def get_options(in_format, bitrate, dtx):
 
 INPUT_FILES = {
     "stereo": "stvST48n.wav",
+    "ISM1": "stv1ISM48s.wav",
+    "ISM2": "stv2ISM48s.wav",
     "ISM3": "stv3ISM48s.wav",
     "ISM4": "stv4ISM48n.wav",
     "MASA1TC": "stv2MASA1TC48c.wav",
     "MASA2TC": "stv2MASA2TC48c.wav",
     "MC_5_1": "stv51MC48c.wav",
+    "MC_5_1_2": "stv512MC48c.wav",
     "MC_5_1_4": "stv514MC48c.wav",
     "MC_7_1": "stv71MC48c.wav",
     "MC_7_1_4": "stv714MC48c.wav",
@@ -100,6 +140,10 @@ INPUT_FILES = {
     "OMASA_ISM4": "stvOMASA_4ISM_2MASA2TC48c.wav",
     "OSBA_ISM2_HOA2": "stvOSBA_2ISM_2OA48c.wav",
     "OSBA_ISM4_HOA2": "stvOSBA_4ISM_2OA48c.wav",
+    "OSBA_ISM4_FOA": "stvOSBA_4ISM_FOA48c.wav",
+    "OSBA_ISM3_HOA3": "stvOSBA_3ISM_3OA48c.wav",
+    "OSBA_ISM2_HOA3": "stvOSBA_2ISM_3OA48c.wav",
+    "OSBA_ISM1_HOA2": "stvOSBA_1ISM_2OA48c.wav",
 }
 
 
-- 
GitLab


From 7a985bd8c3314a8a363489d15c6ed804b3ba7f2e Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Fri, 28 Jun 2024 08:52:42 +0200
Subject: [PATCH 26/30] use test offset in audioarray.compare for all steps

---
 scripts/pyaudio3dtools/audioarray.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/scripts/pyaudio3dtools/audioarray.py b/scripts/pyaudio3dtools/audioarray.py
index 2326d68498..be950aaf60 100644
--- a/scripts/pyaudio3dtools/audioarray.py
+++ b/scripts/pyaudio3dtools/audioarray.py
@@ -265,18 +265,25 @@ def compare(
         Set to True to only apply the threshold comparison for the reference signal
         for whether to include a segment in the ssnr computation. Use this to align
         behaviour with the MPEG-D conformance specification.
-    test_start_offset_ms: int
+    test_start_offset_ms: (non-negative) int
         offset in miliseconds for test signal. If > 0, the corresponding number of samples
-        will be removed from the test array like so: test = test[sample_offset:, :], default 0
+        will be removed from the test array like so: test = test[sample_offset:, :].
 
     Returns
     -------
     result: dict
         Comparison results
     """
-    framesize = fs // 50
+
+    if test_start_offset_ms < 0:
+        raise ValueError(
+            f"Test_start_offset_ms has to be non-negative, but {test_start_offset_ms} was given."
+        )
     test_start_offset_samples = int(fs * test_start_offset_ms / 1000)
-    diff = abs(test[test_start_offset_samples:, :] - ref)
+    test = test[test_start_offset_samples:, :]
+
+    framesize = fs // 50
+    diff = abs(test - ref)
     max_diff = int(diff.max())
     result = {
         "bitexact": True,
-- 
GitLab


From 5dc849b98397debae02bc9e449a0539a4b9cf15e Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Fri, 28 Jun 2024 08:53:47 +0200
Subject: [PATCH 27/30] remove obsolete file

---
 ci/ivas_voip_be_test.sh | 132 ----------------------------------------
 1 file changed, 132 deletions(-)
 delete mode 100755 ci/ivas_voip_be_test.sh

diff --git a/ci/ivas_voip_be_test.sh b/ci/ivas_voip_be_test.sh
deleted file mode 100755
index 9ec6477860..0000000000
--- a/ci/ivas_voip_be_test.sh
+++ /dev/null
@@ -1,132 +0,0 @@
-#! /usr/bin/bash
-
-# (C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB,
-# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
-# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
-# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
-# contributors to this repository. All Rights Reserved.
-
-# This software is protected by copyright law and by international treaties.
-# The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB,
-# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
-# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
-# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
-# contributors to this repository retain full ownership rights in their respective contributions in
-# the software. This notice grants no license of any kind, including but not limited to patent
-# license, nor is any license granted by implication, estoppel or otherwise.
-
-# Contributors are required to enter into the IVAS codec Public Collaboration agreement before making
-# contributions.
-
-# This software is provided "AS IS", without any express or implied warranties. The software is in the
-# development stage. It is intended exclusively for experts who have experience with such software and
-# solely for the purpose of inspection. All implied warranties of non-infringement, merchantability
-# and fitness for a particular purpose are hereby disclaimed and excluded.
-
-# Any dispute, controversy or claim arising under or in relation to providing this software shall be
-# submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in
-# accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and
-# the United Nations Convention on Contracts on the International Sales of Goods.
-
-function usage {
-    echo
-    echo "Usage:"
-    echo "      ivas_voip_be_test.sh [MODE]"
-    echo 
-    echo "      MODE - test (default) or coverage"
-    exit
-}
-
-if [ ! -d "lib_com" ]; then
-    echo "not in root directory! - please run in IVAS root"
-    exit 1
-fi
-
-if [ -z "$1" ] || [ "$1" == "test" ]; then
-    WORKERS=""
-    BUILD=1
-    COVERAGE=0
-elif [ "$1" == "coverage" ]; then
-    WORKERS="-t 1"
-    BUILD=0
-    COVERAGE=1
-else
-    usage
-fi
-
-if [ $BUILD -eq 1 ];then
-    make clean
-    make all -j
-fi
-
-# Configuration
-modes=(
-    'stereo_b32_dtx_wb_cbr'
-    'stereo_b48_dtx_fb_cbr'
-    'ISM4_b48_dtx_fb_cbr'
-    'ISM3_b64_dtx_swb_cbr'
-    'MASA_1TC_b24_4_dtx_wb_cbr'
-    'MASA_2TC_b80_dtx_fb_cbr'
-    'MC_5_1_b128_swb_cbr'
-    'MC_7_1_b96_wb_cbr'
-    'MC_5_1_4_b48_fb_cbr'
-    'MC_7_1_4_b160_fb_cbr'
-    'OMASA_ISM1_2TC_b512_swb_cbr'
-    'OMASA_ISM2_1TC_b24_4_wb_cbr'
-    'OMASA_ISM3_2TC_b80_fb_cbr'
-    'OMASA_ISM4_1TC_b48_swb_cbr'
-    'OSBA_ISM4_HOA2_b512_fb_cbr'
-    'OSBA_ISM2_HOA2_b64_wb_cbr'
-    'HOA3_b64_swb_cbr'
-    'FOA_b256_fb_cbr'
-)
-limit_input_to_x_seconds=30
-verbosity_cmd="-z console"
-
-cfg=./scripts/config/ci_linux.json
-dly_profile=./scripts/dly_error_profiles/dly_error_profile_0.dat
-
-output_dir_default="out"
-output_dir_voip="out_voip"
-
-# Run the same modes in VoIP and non-VoIP mode with a neutral delay profile
-./scripts/runIvasCodec.py $verbosity_cmd -p $cfg $WORKERS -U $limit_input_to_x_seconds -m "${modes[@]}" -o $output_dir_default | tee voip_be_test_output.txt
-mkdir $output_dir_voip
-cp -r $output_dir_default/enc $output_dir_voip/enc
-./scripts/runIvasCodec.py $verbosity_cmd -p $cfg $WORKERS -U $limit_input_to_x_seconds -m "${modes[@]}" -o $output_dir_voip -J "$dly_profile" --decoder_only | tee -a voip_be_test_output.txt
-
-# Check if Python scripts above failed. They return status 0 even when running a mode fails, so we have to parse log file
-if grep -iq failed voip_be_test_output.txt ; then
-    echo "Run errors in runIvasCodec.py"
-    exit 1
-fi
-
-if [ $COVERAGE -eq 1 ];then
-    # Coverage analysis requires only running the codec and may exit before the comparison part
-    exit 0
-fi
-
-# Set up Python path
-python_audio_module_path=$(pwd)/scripts
-export PYTHONPATH=$python_audio_module_path:$PYTHONPATH
-python_audiofile_script_path=$python_audio_module_path/pyaudio3dtools/audiofile.py
-
-jbm_offset=60
-output_dir_default_dec="$output_dir_default"/dec
-output_dir_jbm_dec=${output_dir_default_dec/$output_dir_default/$output_dir_voip}
-
-# strip part of filenames
-for f in $output_dir_jbm_dec/*.wav; do
-    mv $f ${f/_jbm_dly_error_profile_0_dat/}
-done
-
-num_diff=0
-python3 $python_audio_module_path/batch_comp_audio.py $output_dir_default_dec $output_dir_jbm_dec -ds --tool pyaudio3dtools --test_offset_ms $jbm_offset
-num_diff=$?
-
-if [ $num_diff == 0 ]; then
-    printf "\n\nAll tested conditions are bit-exact\n" | tee -a voip_be_test_output.txt
-else
-    printf "\n\nBitexactness problems found!\n" | tee -a voip_be_test_output.txt
-    exit 1;
-fi
-- 
GitLab


From f24cc328c9c0445d68872ea60d780597b22c5b73 Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Fri, 28 Jun 2024 08:57:45 +0200
Subject: [PATCH 28/30] add todos and cleanup in jbm be test script

---
 tests/test_be_for_jbm_neutral_dly_profile.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/tests/test_be_for_jbm_neutral_dly_profile.py b/tests/test_be_for_jbm_neutral_dly_profile.py
index a135181672..15646c773e 100644
--- a/tests/test_be_for_jbm_neutral_dly_profile.py
+++ b/tests/test_be_for_jbm_neutral_dly_profile.py
@@ -14,6 +14,7 @@ DTX_ON = "DTX_ON"
 DTX_OFF = "DTX_OFF"
 
 
+# TODO: Clarify if this should also be tested with DTX, see #1122
 TESTCASES = [
     # stereo
     ["stereo", 32000, "EXT", "DTX_OFF"],
@@ -187,17 +188,14 @@ def test_be_for_jbm_neutral_dly_profile(
         x_jbm, _ = audiofile.readfile(output_jbm)
 
         # strip jbm delay
+        # TODO: this may need to be adapted to handle variable offsets based on outcome of #1122
         delay_samples = JBM_NEUTRAL_DELAY_MS * sampling_rate_khz
-        x_jbm = x_jbm[delay_samples:, :]
-        # TODO: if #1122 results in the JBM output always being longer,
-        # then this should be removed and a length check with failure otherwise added
-        min_len = min(len(x), len(x_jbm))
-
         cmp_result = audioarray.compare(
-            x[:min_len, :],
-            x_jbm[:min_len, :],
+            x,
+            x_jbm,
             fs=sampling_rate_khz * 1000,
             per_frame=False,
+            test_start_offset_ms=delay_samples,
         )
         if not cmp_result["bitexact"]:
             pytest.fail("Difference between no jbm and zero-delay jbm decoding!")
-- 
GitLab


From 2e8dab5d22112b07c499f2efa9b6a1dec739af7c Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Fri, 28 Jun 2024 09:03:29 +0200
Subject: [PATCH 29/30] use new pytest impl in coverage test

---
 .gitlab-ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0dfb40fadd..1b192ec44d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1016,6 +1016,7 @@ voip-be-on-merge-request:
   timeout: "10 minutes"
   script:
     - *print-common-info
+    - make clean
     - make -j
     - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py
 
@@ -2206,7 +2207,7 @@ coverage-test-on-main-scheduled:
     - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec
     - bash ci/smoke_test.sh coverage
     - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 -m create_ref --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec
-    - bash ci/ivas_voip_be_test.sh coverage
+    - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py
     - lcov -c -d obj -o coverage.info
     # remove apps and lib_util files from coverage
     - lcov -r coverage.info "*apps*" -o coverage.info
-- 
GitLab


From 899d33cf3fc10fecb90745e116c885fa77201513 Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Fri, 28 Jun 2024 09:08:48 +0200
Subject: [PATCH 30/30] pass delay value correctly

---
 tests/test_be_for_jbm_neutral_dly_profile.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/test_be_for_jbm_neutral_dly_profile.py b/tests/test_be_for_jbm_neutral_dly_profile.py
index 15646c773e..64e7773a60 100644
--- a/tests/test_be_for_jbm_neutral_dly_profile.py
+++ b/tests/test_be_for_jbm_neutral_dly_profile.py
@@ -189,13 +189,12 @@ def test_be_for_jbm_neutral_dly_profile(
 
         # strip jbm delay
         # TODO: this may need to be adapted to handle variable offsets based on outcome of #1122
-        delay_samples = JBM_NEUTRAL_DELAY_MS * sampling_rate_khz
         cmp_result = audioarray.compare(
             x,
             x_jbm,
             fs=sampling_rate_khz * 1000,
             per_frame=False,
-            test_start_offset_ms=delay_samples,
+            test_start_offset_ms=JBM_NEUTRAL_DELAY_MS,
         )
         if not cmp_result["bitexact"]:
             pytest.fail("Difference between no jbm and zero-delay jbm decoding!")
-- 
GitLab