From fe821a4bbf32f5972b3cdfeb4ef1e731e6293962 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 3 Sep 2025 10:37:48 +0200 Subject: [PATCH 1/5] add mechanism for using DISABLE_HRTF again --- main-basop.yml | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index 207ad3e..9df4e82 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -374,6 +374,10 @@ workflow: before_script: - !reference [ .test-job-linux, before_script ] - rm -rf tests/dut tests/ref + + - pytest_opts="" + - if [ "$DISABLE_HRTF" = "true" ]; then + - pytest_opts="-k not model" variables: USE_LTV: 0 @@ -422,11 +426,11 @@ workflow: - python3 ci/remove_unsupported_testcases.py $PRM_FILES - exit_code_target=0 - - python3 -m pytest $TEST_SUITE -v --update_ref 1 --create_ref -n auto --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH > $PYTEST_LOG_TARGET_BRANCH || exit_code_target=$? + - python3 -m pytest $pytest_opts $TEST_SUITE -v --update_ref 1 --create_ref -n auto --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH > $PYTEST_LOG_TARGET_BRANCH || exit_code_target=$? - exit_code=0 - rm -rf .pytest_cache || true - - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? + - python3 -m pytest $pytest_opts --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? - if [ $exit_code -ne 0 ]; then - exit_code=$EXIT_CODE_NON_BE @@ -567,10 +571,10 @@ workflow: ### ----- run pytest for branch (dut) first ----- # create float reference outputs with corresponding branch - - python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH || exit_code=$? + - python3 -m pytest $pytest_opts $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH || exit_code=$? - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh # create dut outputs - - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true + - python3 -m pytest $pytest_opts --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $XML_REPORT_BRANCH $CSV_BRANCH # Store branch outputs for later comparison @@ -582,10 +586,10 @@ workflow: ### ----- run pytest for merge target now ----- # create float reference outputs with corresponding branch - - python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH || exit_code=$? + - python3 -m pytest $pytest_opts $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH || exit_code=$? - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh # create merge-target outputs - - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --dut_encoder_path $MERGE_TARGET_ENCODER_PATH --dut_decoder_path $MERGE_TARGET_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true + - python3 -m pytest $pytest_opts --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --dut_encoder_path $MERGE_TARGET_ENCODER_PATH --dut_decoder_path $MERGE_TARGET_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN ### compare the two csv files for regressions @@ -1310,9 +1314,13 @@ ivas-pytest-on-merge-request: ### If ref_using_target is not set, checkout the source branch to use scripts and input from there - if [ $ref_using_target == 0 ]; then git checkout $source_branch_commit_sha; fi + - pytest_opts="" + - if [ "$DISABLE_HRTF" = "true" ]; then + - pytest_opts="-k not model" + ### prepare pytest # create references - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 + - python3 -m pytest $pytest_opts $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 ### Run test using branch scripts and input - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi @@ -1320,7 +1328,7 @@ ivas-pytest-on-merge-request: ### run pytest - exit_code=0 - testcase_timeout=600 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest $pytest_opts $TESTS_DIR_CODEC_BE_ON_MR -v --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV -- GitLab From 94d9d1793b83b380be82dbf924a5a028b340c0c9 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 3 Sep 2025 10:58:58 +0200 Subject: [PATCH 2/5] Change mechanism to use PYTEST_ADDOPTS instead Using '' inside the env var worked locally, let's see... --- main-basop.yml | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index 9df4e82..838d032 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -374,10 +374,9 @@ workflow: before_script: - !reference [ .test-job-linux, before_script ] - rm -rf tests/dut tests/ref - - - pytest_opts="" - if [ "$DISABLE_HRTF" = "true" ]; then - - pytest_opts="-k not model" + - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'" + - fi variables: USE_LTV: 0 @@ -426,11 +425,11 @@ workflow: - python3 ci/remove_unsupported_testcases.py $PRM_FILES - exit_code_target=0 - - python3 -m pytest $pytest_opts $TEST_SUITE -v --update_ref 1 --create_ref -n auto --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH > $PYTEST_LOG_TARGET_BRANCH || exit_code_target=$? + - python3 -m pytest $TEST_SUITE -v --update_ref 1 --create_ref -n auto --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH > $PYTEST_LOG_TARGET_BRANCH || exit_code_target=$? - exit_code=0 - rm -rf .pytest_cache || true - - python3 -m pytest $pytest_opts --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? + - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? - if [ $exit_code -ne 0 ]; then - exit_code=$EXIT_CODE_NON_BE @@ -571,10 +570,10 @@ workflow: ### ----- run pytest for branch (dut) first ----- # create float reference outputs with corresponding branch - - python3 -m pytest $pytest_opts $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH || exit_code=$? + - python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH || exit_code=$? - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh # create dut outputs - - python3 -m pytest $pytest_opts --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true + - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $XML_REPORT_BRANCH $CSV_BRANCH # Store branch outputs for later comparison @@ -586,10 +585,10 @@ workflow: ### ----- run pytest for merge target now ----- # create float reference outputs with corresponding branch - - python3 -m pytest $pytest_opts $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH || exit_code=$? + - python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH || exit_code=$? - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh # create merge-target outputs - - python3 -m pytest $pytest_opts --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --dut_encoder_path $MERGE_TARGET_ENCODER_PATH --dut_decoder_path $MERGE_TARGET_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true + - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --dut_encoder_path $MERGE_TARGET_ENCODER_PATH --dut_decoder_path $MERGE_TARGET_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN ### compare the two csv files for regressions @@ -1314,13 +1313,13 @@ ivas-pytest-on-merge-request: ### If ref_using_target is not set, checkout the source branch to use scripts and input from there - if [ $ref_using_target == 0 ]; then git checkout $source_branch_commit_sha; fi - - pytest_opts="" - if [ "$DISABLE_HRTF" = "true" ]; then - - pytest_opts="-k not model" + - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'" + - fi ### prepare pytest # create references - - python3 -m pytest $pytest_opts $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 ### Run test using branch scripts and input - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi @@ -1328,7 +1327,7 @@ ivas-pytest-on-merge-request: ### run pytest - exit_code=0 - testcase_timeout=600 - - python3 -m pytest $pytest_opts $TESTS_DIR_CODEC_BE_ON_MR -v --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV -- GitLab From ffe50a9b38b8fe643b9cd424d5dd7fa06bc9ca24 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 3 Sep 2025 11:44:26 +0200 Subject: [PATCH 3/5] set default for PYTEST_ADDOPTS --- includes/default-variables.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/includes/default-variables.yml b/includes/default-variables.yml index 3209ec5..f4b0f60 100644 --- a/includes/default-variables.yml +++ b/includes/default-variables.yml @@ -12,3 +12,4 @@ variables: BASOP_REFERENCE_BRANCH: "ivas-float-update" SCALE_FACTOR: "3.162" BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch" + PYTEST_ADDOPTS: "" -- GitLab From 3442f10ad8b97835b33b788b0da061b8a7e74e19 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 3 Sep 2025 14:32:08 +0200 Subject: [PATCH 4/5] move default for DISABLE_HRTF from BASOP repo to here --- includes/default-variables.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/includes/default-variables.yml b/includes/default-variables.yml index f4b0f60..ac274be 100644 --- a/includes/default-variables.yml +++ b/includes/default-variables.yml @@ -13,3 +13,5 @@ variables: SCALE_FACTOR: "3.162" BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch" PYTEST_ADDOPTS: "" + # set this to true to skip the external HRTF testcases in pytest calls + DISABLE_HRTF: "false" -- GitLab From ba113a378b7ff8bcfd676612a845f5670b80b048 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 3 Sep 2025 14:35:24 +0200 Subject: [PATCH 5/5] change comment --- includes/default-variables.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/includes/default-variables.yml b/includes/default-variables.yml index ac274be..2774c57 100644 --- a/includes/default-variables.yml +++ b/includes/default-variables.yml @@ -13,5 +13,5 @@ variables: SCALE_FACTOR: "3.162" BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch" PYTEST_ADDOPTS: "" - # set this to true to skip the external HRTF testcases in pytest calls + # overwrite this via .gitlab/variables.yml in the respective other repo to disable "(model from file)" testcases DISABLE_HRTF: "false" -- GitLab