From d535bd11d698a6837b531a580c96110ee455f533 Mon Sep 17 00:00:00 2001 From: norvell Date: Fri, 16 Sep 2022 17:15:46 +0000 Subject: [PATCH 1/8] Added self_test_evs.prm check, allowing differences if [EVS Non-BE] is specified. --- .gitlab-ci.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 624322e87b..244fb8e3b6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -298,13 +298,15 @@ self-test-on-merge-request: ### run selftest - ls -altr scripts/testv - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test | tee test_output.txt - + - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test scripts/config/self_test_evs.psm | tee test_output_evs.txt ### analyse test output # some helper variables - "|| true" to prevent failures from grep not finding anything + - evs_non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[evs[ -]non[ -]*be\]") || true - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true - run_errors=$(cat test_output.txt | grep -c "test conditions had run errors") || true - bitexact=$(cat test_output.txt | grep -c "All [0-9]* tests are bitexact") || true + - bitexact_evs=$(cat test_output_evs.txt | grep -c "All [0-9]* tests are bitexact") || true - EXIT_CODE_NON_BE=123 - EXIT_CODE_FAIL=1 @@ -313,9 +315,13 @@ self-test-on-merge-request: # check for crashes during the test, if any happened, fail the test - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; exit $EXIT_CODE_FAIL; fi + # check for non bitexact EVS output + - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag == 0 ] ; then echo "Non-bitexact EVS cases without EVS-non-BE tag encountered"; selftest_evs_exit_code=$EXIT_CODE_FAIL; fi + - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag != 0 ] ; then echo "Non-bitexact EVS cases with EVS-non-BE tag encountered"; selftest_evs_exit_code=$EXIT_CODE_NON_BE; fi + # check for non bitexact output and store exit code to also always run the SBA pytest - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_FAIL; fi - - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_NON_BE; fi + - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; selftest_exit_code=selftest_evs_exit_code; fi ### run SBA pytest - exit_code=0 -- GitLab From 6a58fa425802a5f8169a9e9d51d85af6a06c8156 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 16 Sep 2022 19:29:44 +0200 Subject: [PATCH 2/8] Updated .gitlab-ci.yml to run EVS self_test. Correct bug in case of failed self_test.py and self_test_b.py passes --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 244fb8e3b6..d020b99227 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -321,14 +321,14 @@ self-test-on-merge-request: # check for non bitexact output and store exit code to also always run the SBA pytest - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_FAIL; fi - - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; selftest_exit_code=selftest_evs_exit_code; fi + - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; selftest_exit_code=$selftest_evs_exit_code; fi ### run SBA pytest - exit_code=0 - python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$? - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi + - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $selftest_exit_code; fi - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi; # return exit code from selftest if everything went well with the pytest run - exit $selftest_exit_code -- GitLab From 18c9e310b12b1dc4610f839ca42ea6d48d499ffc Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 16 Sep 2022 22:41:18 +0200 Subject: [PATCH 3/8] Corrected spelling error on self_test_evs.prm --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d020b99227..dcc553bbcb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -298,7 +298,7 @@ self-test-on-merge-request: ### run selftest - ls -altr scripts/testv - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test | tee test_output.txt - - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test scripts/config/self_test_evs.psm | tee test_output_evs.txt + - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test scripts/config/self_test_evs.prm | tee test_output_evs.txt ### analyse test output # some helper variables - "|| true" to prevent failures from grep not finding anything -- GitLab From a7e51ba7b735d0051c684713be69bdbd89e55b05 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 16 Sep 2022 22:48:49 +0200 Subject: [PATCH 4/8] Added test_output_evs.txt artifact --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dcc553bbcb..3059664d00 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -340,6 +340,7 @@ self-test-on-merge-request: when: always paths: - test_output.txt + - test_output_evs.txt - scripts/test/logs/ - scripts/ref/logs/ - report-junit.xml -- GitLab From 776681f6c483c008b79356c3b05f8dcd9f160bb8 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 16 Sep 2022 22:56:02 +0200 Subject: [PATCH 5/8] Included test_output_evs.txt in run error check --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3059664d00..b958f3a321 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -304,7 +304,7 @@ self-test-on-merge-request: # some helper variables - "|| true" to prevent failures from grep not finding anything - evs_non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[evs[ -]non[ -]*be\]") || true - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true - - run_errors=$(cat test_output.txt | grep -c "test conditions had run errors") || true + - run_errors=$(cat test_output.txt test_output_evs.txt | grep -c "test conditions had run errors") || true - bitexact=$(cat test_output.txt | grep -c "All [0-9]* tests are bitexact") || true - bitexact_evs=$(cat test_output_evs.txt | grep -c "All [0-9]* tests are bitexact") || true - EXIT_CODE_NON_BE=123 -- GitLab From 18acab3bd9ce397eef49f11bf85a7b68e0a13f3e Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Sat, 17 Sep 2022 19:12:26 +0200 Subject: [PATCH 6/8] Removed JBM cases from self_test_evs.prm -- networkSimulator_g192 not available. --- scripts/config/self_test_evs.prm | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/scripts/config/self_test_evs.prm b/scripts/config/self_test_evs.prm index 02227154d8..4f6908be4c 100644 --- a/scripts/config/self_test_evs.prm +++ b/scripts/config/self_test_evs.prm @@ -219,18 +219,3 @@ ../IVAS_dec 32 bit testv/stv32c_AMR_WB_19k85_32-32.tst - -// Codec A at 13.20 kbps, 32kHz in, 32kHz out, DTX, JBM Prof 5 -../IVAS_cod -dtx 13200 32 testv/stv32c.pcm bit -networkSimulator_g192 ../scripts/dly_error_profiles/dly_error_profile_5.dat bit netsimoutput tracefile_sim 2 0 -../IVAS_dec -Tracefile tracefile_dec -VOIP 32 netsimoutput testv/stv32c_13k20_32-32_DTX_JBM5.tst - -// Codec B at 16.40 kbps, 32kHz in, 32kHz out, DTX, JBM Prof 5 -../IVAS_cod -dtx 16400 32 testv/stv32c.pcm bit -networkSimulator_g192 ../scripts/dly_error_profiles/dly_error_profile_5.dat bit netsimoutput tracefile_sim 2 0 -../IVAS_dec -Tracefile tracefile_dec -VOIP 32 netsimoutput testv/stv32c_16k40_32-32_DTX_JBM5.tst - -// Codec B at 13.20 kbps, 32kHz in, 32kHz out, JBM Prof 9, Channel aware -../IVAS_cod -rf 13200 32 testv/stv32c.pcm bit -networkSimulator_g192 ../scripts/dly_error_profiles/dly_error_profile_5.dat bit netsimoutput tracefile_sim 2 0 -../IVAS_dec -Tracefile tracefile_dec -VOIP 32 netsimoutput testv/stv32c_13k20_CA_32-32_JBM9.tst -- GitLab From 8e21881eec1de2342d9c8dde7faea5a52b5b41b9 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Sun, 18 Sep 2022 10:37:16 +0200 Subject: [PATCH 7/8] Corrected logic for the three subtests --- .gitlab-ci.yml | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b958f3a321..0a8acac9f9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -309,29 +309,37 @@ self-test-on-merge-request: - bitexact_evs=$(cat test_output_evs.txt | grep -c "All [0-9]* tests are bitexact") || true - EXIT_CODE_NON_BE=123 - EXIT_CODE_FAIL=1 - - - selftest_exit_code=0 + - expected_nonbe_1=0 + - expected_nonbe_2=0 + - expected_nonbe_3=0 + - fail_1=0 + - fail_2=0 + - fail_3=0 # check for crashes during the test, if any happened, fail the test - - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; exit $EXIT_CODE_FAIL; fi - - # check for non bitexact EVS output - - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag == 0 ] ; then echo "Non-bitexact EVS cases without EVS-non-BE tag encountered"; selftest_evs_exit_code=$EXIT_CODE_FAIL; fi - - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag != 0 ] ; then echo "Non-bitexact EVS cases with EVS-non-BE tag encountered"; selftest_evs_exit_code=$EXIT_CODE_NON_BE; fi + - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; fail_1=$EXIT_CODE_FAIL; fi # check for non bitexact output and store exit code to also always run the SBA pytest - - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_FAIL; fi - - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; selftest_exit_code=$selftest_evs_exit_code; fi + - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; fail_1=1; fi + - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; expected_nonbe_1=1; fi + + # check for non bitexact EVS output + - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag == 0 ] ; then echo "Non-bitexact EVS cases without EVS-non-BE tag encountered"; fail_2=1; fi + - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag != 0 ] ; then echo "Non-bitexact EVS cases with EVS-non-BE tag encountered"; expected_nonbe_2=1; fi ### run SBA pytest - exit_code=0 - python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$? - - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; fail_3=1; fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $selftest_exit_code; fi - - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi; - # return exit code from selftest if everything went well with the pytest run - - exit $selftest_exit_code + - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; expected_nonbe_3=1; fi + - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; fail_3=1; fi; + + # Check results from all three tests + - if [ $fail_1 -eq 1 ] || [ $fail_2 -eq 1 ] || [ $fail_3 -eq 1 ]; then exit $EXIT_CODE_FAIL; fi + - if [ $expected_nonbe_1 -eq 1 ] || [ $expected_nonbe_2 -eq 1 ] || [ $expected_nonbe_3 -eq 1 ]; then exit $EXIT_CODE_NON_BE; fi + - exit 0 + allow_failure: exit_codes: - 123 -- GitLab From 2c440902e78fbdd9ad01e1c7d05dc1cb5611b2af Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Mon, 19 Sep 2022 14:24:34 +0200 Subject: [PATCH 8/8] Corrected tests in self-test-on-merge-request for self_test_b.py --- .gitlab-ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0a8acac9f9..0aeb3ceec9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -317,7 +317,7 @@ self-test-on-merge-request: - fail_3=0 # check for crashes during the test, if any happened, fail the test - - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; fail_1=$EXIT_CODE_FAIL; fi + - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; fail_1=1; fi # check for non bitexact output and store exit code to also always run the SBA pytest - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; fail_1=1; fi @@ -330,10 +330,12 @@ self-test-on-merge-request: ### run SBA pytest - exit_code=0 - python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$? - - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; fail_3=1; fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; expected_nonbe_3=1; fi - - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; fail_3=1; fi; + + - if [ $zero_errors != 1 ]; then echo "Run errors in SBA pytest"; fail_3=1; fi + + - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures without non-BE tag encountered"; fail_3=1; fi + - if [ $exit_code -eq 1 ] && [ $non_be_flag == 1 ]; then echo "pytest run had failures with non-BE tag encountered"; expected_nonbe_3=1; fi # Check results from all three tests - if [ $fail_1 -eq 1 ] || [ $fail_2 -eq 1 ] || [ $fail_3 -eq 1 ]; then exit $EXIT_CODE_FAIL; fi -- GitLab