diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2ac6686350b31752cad6d27a449d1890767e48c2..bdfefff19d82915daddb6e72ab6b3779a24318a6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -301,34 +301,50 @@ self-test-on-merge-request: ### run selftest - ls -altr scripts/testv - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test | tee test_output.txt - + - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test scripts/config/self_test_evs.prm | tee test_output_evs.txt ### analyse test output # some helper variables - "|| true" to prevent failures from grep not finding anything + - evs_non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[evs[ -]non[ -]*be\]") || true - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true - - run_errors=$(cat test_output.txt | grep -c "test conditions had run errors") || true + - run_errors=$(cat test_output.txt test_output_evs.txt | grep -c "test conditions had run errors") || true - bitexact=$(cat test_output.txt | grep -c "All [0-9]* tests are bitexact") || true + - bitexact_evs=$(cat test_output_evs.txt | grep -c "All [0-9]* tests are bitexact") || true - EXIT_CODE_NON_BE=123 - EXIT_CODE_FAIL=1 - - - selftest_exit_code=0 + - expected_nonbe_1=0 + - expected_nonbe_2=0 + - expected_nonbe_3=0 + - fail_1=0 + - fail_2=0 + - fail_3=0 # check for crashes during the test, if any happened, fail the test - - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; exit $EXIT_CODE_FAIL; fi + - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; fail_1=1; fi # check for non bitexact output and store exit code to also always run the SBA pytest - - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_FAIL; fi - - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_NON_BE; fi + - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; fail_1=1; fi + - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; expected_nonbe_1=1; fi + + # check for non bitexact EVS output + - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag == 0 ] ; then echo "Non-bitexact EVS cases without EVS-non-BE tag encountered"; fail_2=1; fi + - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag != 0 ] ; then echo "Non-bitexact EVS cases with EVS-non-BE tag encountered"; expected_nonbe_2=1; fi ### run SBA pytest - exit_code=0 - python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$? - - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi - - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi; - # return exit code from selftest if everything went well with the pytest run - - exit $selftest_exit_code + + - if [ $zero_errors != 1 ]; then echo "Run errors in SBA pytest"; fail_3=1; fi + + - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures without non-BE tag encountered"; fail_3=1; fi + - if [ $exit_code -eq 1 ] && [ $non_be_flag == 1 ]; then echo "pytest run had failures with non-BE tag encountered"; expected_nonbe_3=1; fi + + # Check results from all three tests + - if [ $fail_1 -eq 1 ] || [ $fail_2 -eq 1 ] || [ $fail_3 -eq 1 ]; then exit $EXIT_CODE_FAIL; fi + - if [ $expected_nonbe_1 -eq 1 ] || [ $expected_nonbe_2 -eq 1 ] || [ $expected_nonbe_3 -eq 1 ]; then exit $EXIT_CODE_NON_BE; fi + - exit 0 + allow_failure: exit_codes: - 123 @@ -337,6 +353,7 @@ self-test-on-merge-request: when: always paths: - test_output.txt + - test_output_evs.txt - scripts/test/logs/ - scripts/ref/logs/ - report-junit.xml diff --git a/scripts/config/self_test_evs.prm b/scripts/config/self_test_evs.prm index 02227154d895bb542b41643a6359a162d03ecb53..4f6908be4c8387a61a5b93db1a11b0b14fbee94d 100644 --- a/scripts/config/self_test_evs.prm +++ b/scripts/config/self_test_evs.prm @@ -219,18 +219,3 @@ ../IVAS_dec 32 bit testv/stv32c_AMR_WB_19k85_32-32.tst - -// Codec A at 13.20 kbps, 32kHz in, 32kHz out, DTX, JBM Prof 5 -../IVAS_cod -dtx 13200 32 testv/stv32c.pcm bit -networkSimulator_g192 ../scripts/dly_error_profiles/dly_error_profile_5.dat bit netsimoutput tracefile_sim 2 0 -../IVAS_dec -Tracefile tracefile_dec -VOIP 32 netsimoutput testv/stv32c_13k20_32-32_DTX_JBM5.tst - -// Codec B at 16.40 kbps, 32kHz in, 32kHz out, DTX, JBM Prof 5 -../IVAS_cod -dtx 16400 32 testv/stv32c.pcm bit -networkSimulator_g192 ../scripts/dly_error_profiles/dly_error_profile_5.dat bit netsimoutput tracefile_sim 2 0 -../IVAS_dec -Tracefile tracefile_dec -VOIP 32 netsimoutput testv/stv32c_16k40_32-32_DTX_JBM5.tst - -// Codec B at 13.20 kbps, 32kHz in, 32kHz out, JBM Prof 9, Channel aware -../IVAS_cod -rf 13200 32 testv/stv32c.pcm bit -networkSimulator_g192 ../scripts/dly_error_profiles/dly_error_profile_5.dat bit netsimoutput tracefile_sim 2 0 -../IVAS_dec -Tracefile tracefile_dec -VOIP 32 netsimoutput testv/stv32c_13k20_CA_32-32_JBM9.tst