diff --git a/includes/default-variables.yml b/includes/default-variables.yml index 2774c578ba79b4182014d9a7071649b110dae633..9b8acc40cda41fb0f001cefd9ae3c2b456dad9d9 100644 --- a/includes/default-variables.yml +++ b/includes/default-variables.yml @@ -11,7 +11,7 @@ variables: TESTCASE_TIMEOUT_LTV_SANITIZERS: 2400 BASOP_REFERENCE_BRANCH: "ivas-float-update" SCALE_FACTOR: "3.162" - BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch" + BASOP_CI_BRANCH_PC_REPO: "ci/add-inplace-option-to-create-short-testvectory-script" PYTEST_ADDOPTS: "" # overwrite this via .gitlab/variables.yml in the respective other repo to disable "(model from file)" testcases DISABLE_HRTF: "false" diff --git a/main-basop.yml b/main-basop.yml index 6cc465c74d9f9f49f0eb2dcfc356208a7e708c3f..39f9bfa7c7e1adafd2392d2276afd4ef6679e995 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1718,8 +1718,10 @@ renderer-usan: .sanitizer-selftest-on-mr: stage: test + needs: ["build-codec-sanitizers-linux"] extends: - - .rules-merge-request-to-float-ref-no-draft + - .rules-merge-request-no-draft + - .test-job-linux artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week @@ -1742,73 +1744,91 @@ renderer-usan: # to be reused in MR and LTV-scheduled sanitizer test jobs # set CLANG_NUM, SELFTEST_SANITY_TIMEOUT and SELF_TEST_PRM_FILE in before_script section .sanitizer-selftest-anchor: &sanitizer-selftest-anchor - script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - - make clean - - make -j CLANG=$CLANG_NUM - testcase_timeout=$SELFTEST_SANITY_TIMEOUT - export UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 - python3 ci/remove_unsupported_testcases.py $PRM_FILES + - make_args="CLANG=$CLANG_NUM" + # for MRs to main, the whole sanitizer test takes too long (even with stv set of inputs) + # shorten files to 1s for that case + - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "main" ]; then + - python3 tests/create_short_testvectors.py --cut_len 1.0 --inplace --all + + # also need different setup for ubsan + - if [[ $CLANG_NUM == 3 ]]; then + - export UBSAN_OPTIONS="suppressions=scripts/ubsan_basop.supp,report_error_type=1,print_stacktrace=1" + - python3 scripts/basop_create_ignorelist_for_ubsan.py + - make_args="$make_args IGNORELIST=1" + - fi + - fi + - exit_code20=0 - exit_code10=0 - exit_code5=0 - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi + - make clean + - make -j $make_args + # run encoder and decoder with 20ms renderer framesize first, use reference creation mode - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? - # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option - # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active - # we are only interested in runtime errors from the sanitizers and ignore the diffs - - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? - - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? + + # check different renderer fremsizes only for float + - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" != "main" ]; then + # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option + # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active + # we are only interested in runtime errors from the sanitizers and ignore the diffs + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? + - fi - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi # from float repo # NOTE: (per-testcase) timeout was adjusted to work for float only # code selftest testvectors with memory-sanitizer binaries -.codec-msan: +codec-msan: extends: - .sanitizer-selftest-on-mr tags: - ivas-basop-linux - before_script: + script: - CLANG_NUM=1 - SELFTEST_SANITY_TIMEOUT=180 - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" - USE_LTV="" - <<: *sanitizer-selftest-anchor + - *sanitizer-selftest-anchor # code selftest testvectors with address-sanitizer binaries -.codec-asan: +codec-asan: extends: - .sanitizer-selftest-on-mr tags: - ivas-basop-linux - before_script: + script: - CLANG_NUM=2 - SELFTEST_SANITY_TIMEOUT=180 - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" - USE_LTV="" - <<: *sanitizer-selftest-anchor + - *sanitizer-selftest-anchor # code selftest testvectors with undefined-behaviour-sanitizer binaries -.codec-usan: +codec-usan: extends: - .sanitizer-selftest-on-mr tags: - ivas-basop-linux - before_script: + script: - CLANG_NUM=3 - SELFTEST_SANITY_TIMEOUT=180 - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" - USE_LTV="" - <<: *sanitizer-selftest-anchor + - *sanitizer-selftest-anchor # --------------------------------------------------------------- # Short test jobs for running from web interface or schedule @@ -1891,39 +1911,6 @@ ivas-pytest-compare_to_ref-dmx-short-enc-lev+10: - LEVEL_SCALING=3.162 <<: *ivas-pytest-anchor -ivas-pytest-enc-msan: - extends: - - .test-job-linux - tags: - - ivas-basop-linux-fast - before_script: - - CLANG_NUM=1 - - DUT_DECODER_PATH=./$REF_DECODER_PATH - - TEST_SUITE=$LONG_TEST_SUITE_ENCODER - <<: *ivas-pytest-sanitizers-anchor - -ivas-pytest-enc-asan: - extends: - - .test-job-linux - tags: - - ivas-basop-linux-fast - before_script: - - CLANG_NUM=2 - - DUT_DECODER_PATH=./$REF_DECODER_PATH - - TEST_SUITE=$LONG_TEST_SUITE_ENCODER - <<: *ivas-pytest-sanitizers-anchor - -ivas-pytest-enc-usan: - extends: - - .test-job-linux - tags: - - ivas-basop-linux-fast - before_script: - - CLANG_NUM=3 - - DUT_DECODER_PATH=./$REF_DECODER_PATH - - TEST_SUITE=$LONG_TEST_SUITE_ENCODER - <<: *ivas-pytest-sanitizers-anchor - ### jobs that test flt encoder -> fx decoder ivas-pytest-compare_to_ref-short-dec: extends: @@ -1981,39 +1968,6 @@ ivas-pytest-compare-to-input-short-enc: - LEVEL_SCALING=1.0 <<: *ivas-pytest-compare-to-input-anchor -ivas-pytest-dec-msan: - extends: - - .test-job-linux - tags: - - ivas-basop-linux-fast - before_script: - - CLANG_NUM=1 - - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER - <<: *ivas-pytest-sanitizers-anchor - -ivas-pytest-dec-asan: - extends: - - .test-job-linux - tags: - - ivas-basop-linux-fast - before_script: - - CLANG_NUM=2 - - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER - <<: *ivas-pytest-sanitizers-anchor - -ivas-pytest-dec-usan: - extends: - - .test-job-linux - tags: - - ivas-basop-linux-fast - before_script: - - CLANG_NUM=3 - - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER - <<: *ivas-pytest-sanitizers-anchor - # --------------------------------------------------------------- # Long test job # --------------------------------------------------------------- @@ -2141,6 +2095,38 @@ ivas-pytest-compare_ref-long-fx-fx-lev+10: - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor +### These next 3 jobs run three different sanitizers for fx -> fx codec chain usin + +ivas-pytest-msan: + extends: + - .test-job-linux + tags: + - ivas-basop-linux-fast + before_script: + - CLANG_NUM=1 + - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER + <<: *ivas-pytest-sanitizers-anchor + +ivas-pytest-asan: + extends: + - .test-job-linux + tags: + - ivas-basop-linux-fast + before_script: + - CLANG_NUM=2 + - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER + <<: *ivas-pytest-sanitizers-anchor + +ivas-pytest-usan: + extends: + - .test-job-linux + tags: + - ivas-basop-linux-fast + before_script: + - CLANG_NUM=3 + - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER + <<: *ivas-pytest-sanitizers-anchor + # Manual long self-test (not including tests_sba.py) against BASOP main test-long-self-test: extends: