diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f41d978398d2f12d4043fda7a7a8bfae366ab4ed..67e905a6c8c05283fbd8d73646c860d8428328b3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,6 +8,7 @@ variables: EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 LONG_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/renderer --param_file scripts/config/self_test_ltv.prm --use_ltv" + LONG_TEST_SUITE_NO_RENDERER: "tests/codec_be_on_mr_nonselection --param_file scripts/config/self_test_ltv.prm --use_ltv" SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" TEST_SUITE: "" DUT_ENCODER_PATH: "./IVAS_cod" @@ -40,6 +41,7 @@ variables: - 'sanitizer-stv' - 'pytest-renderer' - 'complexity' + - 'coverage' - 'voip-be-test' @@ -115,20 +117,25 @@ stages: echo "Commit time was $CI_COMMIT_TIMESTAMP" date | xargs echo "System time is" -.build-reference-and-dut-binaries: &build-reference-and-dut-binaries - - current_commit_sha=$(git rev-parse HEAD) - ### build reference binaries - - git checkout $REFERENCE_BRANCH - - git pull - - make clean - - make -j - - mv ./IVAS_cod ./$REF_ENCODER_PATH - - mv ./IVAS_dec ./$REF_DECODER_PATH - - mv ./IVAS_rend ./IVAS_rend_ref +.build-reference-binaries: &build-reference-binaries +- current_commit_sha=$(git rev-parse HEAD) +### build reference binaries +- git checkout $REFERENCE_BRANCH +- git pull +- make clean +- make -j +- mv ./IVAS_cod ./$REF_ENCODER_PATH +- mv ./IVAS_dec ./$REF_DECODER_PATH +- mv ./IVAS_rend ./IVAS_rend_ref +### Return to current branch +- git restore . +- git checkout $current_commit_sha - ### build test binaries - - git restore . - - git checkout $current_commit_sha + +.build-reference-and-dut-binaries: &build-reference-and-dut-binaries +### build reference binaries + - *build-reference-binaries +### build dut binaries - make clean - make -j @@ -216,6 +223,15 @@ stages: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' when: never +.rules-coverage: + rules: + - if: $COVERAGE_TEST # Set by scheduled pipeline + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "coverage" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + when: never + .rules-pytest-saturation-smoke-test: rules: - if: $PYTEST_SMOKE_TEST # Set by scheduled pipeline @@ -1000,6 +1016,60 @@ ivas-smoke-test-saturation: - smoke_test_output_hrtf.txt expose_as: "saturation smoke test results" + +# GCOV/LCOV coverage analysis of self_test suite +coverage-test-on-main-scheduled: + extends: + - .test-job-linux + - .rules-coverage + stage: test + timeout: 3 hours + script: + - *print-common-info + - *update-scripts-repo + - *update-ltv-repo + - *copy-ltv-files-to-testv-dir + - *build-reference-binaries + # Build DuT binaries with GCOV + - make clean + - make GCOV=1 -j + - cp IVAS_rend IVAS_rend_ref # Copy to ensure instrumented renderer is run in the first pytest call + + - testcase_timeout=$TESTCASE_TIMEOUT_LTV + - exit_code_dec=0 + - exit_code_enc=0 + - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection tests/renderer --update_ref 1 -v --create_ref --html=report-dec.html --self-contained-html --junit-xml=report-junit-dec.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH || exit_code_dec=$? + - lcov -c -d obj -o coverage_dec_rend.info # extract coverage of decoder/renderer + + - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection --encoder_only -v --html=report-enc.html --self-contained-html --junit-xml=report-junit-enc.xml -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $DUT_ENCODER_PATH || exit_code_enc=$? + - lcov -c -d obj -o coverage_enc_dec_rend.info # extract coverage of encoder/decoder/renderer + + # remove apps and lib_util files from coverage + - lcov -r coverage_dec_rend.info "*apps*" -o coverage_dec_rend.info + - lcov -r coverage_dec_rend.info "*lib_util*" -o coverage_dec_rend.info + - lcov -r coverage_enc_dec_rend.info "*apps*" -o coverage_enc_dec_rend.info + - lcov -r coverage_enc_dec_rend.info "*lib_util*" -o coverage_enc_dec_rend.info + + - commit_sha=$(git rev-parse HEAD) + - genhtml coverage_enc_dec_rend.info -o coverage_enc_dec_rend -t "Coverage on main enc/dec/rend @ $commit_sha" + - genhtml coverage_dec_rend.info -o coverage_dec_rend -t "Coverage on main -- dec/rend @ $commit_sha" + artifacts: + name: "main-coverage-sha-$CI_COMMIT_SHORT_SHA" + when: always + expire_in: 1 week + paths: + - coverage_enc_dec_rend.info + - coverage_dec_rend.info + - coverage_enc_dec_rend + - coverage_dec_rend + - report-dec.html + - report-enc.html + expose_as: "Coverage result" + reports: + junit: + - report-junit-dec.xml + - report-junit-enc.xml + # --------------------------------------------------------------- # EVS 26.444 test job # ---------------------------------------------------------------