From c63fbdedea0467686a4d37f96f8a025fc5d90ffd Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 21 Nov 2022 12:10:22 +0100 Subject: [PATCH 1/4] also generate and archive html report for pytest jobs --- .gitlab-ci.yml | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b9d42a2f85..2a322d6632 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -413,6 +413,9 @@ external-renderer-pytest-on-merge-request: extends: - .test-job-linux - .rules-merge-request + # TODO: remove, only for testing + tags: + - test-fhg-linux-runner1 needs: ["build-codec-linux-make"] # TODO: set reasonable timeout, will most likely take less timeout: "20 minutes" @@ -443,7 +446,7 @@ external-renderer-pytest-on-merge-request: # run test - exit_code=0 - - python3 -m pytest -q --log-level ERROR -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer_be_comparison.py || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --junit-xml=report-junit.xml tests/renderer/test_renderer_be_comparison.py || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check @@ -456,6 +459,7 @@ external-renderer-pytest-on-merge-request: when: always paths: - report-junit.xml + - report.html expose_as: "pytest external renderer results" reports: junit: @@ -466,6 +470,9 @@ ivas-pytest-on-merge-request: extends: - .test-job-linux - .rules-merge-request + # TODO: remove, only for testing + tags: + - test-fhg-linux-runner1 stage: compare needs: ["build-codec-linux-cmake", "codec-smoke-test"] timeout: "10 minutes" @@ -492,7 +499,7 @@ ivas-pytest-on-merge-request: ### run pytest - exit_code=0 - - python3 -m pytest tests -v --junit-xml=report-junit.xml || exit_code=$? + - python3 -m pytest tests -v --html=report.html --junit-xml=report-junit.xml || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check @@ -505,6 +512,7 @@ ivas-pytest-on-merge-request: when: always paths: - report-junit.xml + - report.html expose_as: "pytest ivas results" reports: junit: @@ -514,6 +522,9 @@ evs-pytest-on-merge-request: extends: - .test-job-linux - .rules-merge-request + # TODO: remove, only for testing + tags: + - test-fhg-linux-runner1 stage: compare needs: ["build-codec-linux-cmake", "codec-smoke-test"] timeout: "10 minutes" @@ -537,7 +548,7 @@ evs-pytest-on-merge-request: ### run pytest for EVS cases - exit_code=0 - - python3 -m pytest tests/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --junit-xml=report-junit-evs.xml || exit_code=$? + - python3 -m pytest tests/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --html=report.html --junit-xml=report-junit-evs.xml || exit_code=$? - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check @@ -550,6 +561,7 @@ evs-pytest-on-merge-request: when: always paths: - report-junit-evs.xml + - report.html expose_as: "pytest evs results" reports: junit: @@ -634,6 +646,9 @@ codec-comparison-on-main-push: extends: - .test-job-linux - .rules-main-push + # TODO: remove, only for testing + tags: + - test-fhg-linux-runner1 stage: compare needs: ["build-codec-linux-cmake"] timeout: "30 minutes" # To be revisited @@ -691,7 +706,7 @@ codec-comparison-on-main-push: ### run pytest - exit_code=0 - - python3 -m pytest tests -v --junit-xml=report-junit.xml || exit_code=$? + - python3 -m pytest tests -v --html=report.html --junit-xml=report-junit.xml || exit_code=$? - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi @@ -704,6 +719,7 @@ codec-comparison-on-main-push: when: always paths: - report-junit.xml + - report.html expose_as: "Results of comparison to previous merge commit" reports: junit: report-junit.xml -- GitLab From 318c9e5bf7678eeee2a2d6c133bb89c7242b6038 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 22 Nov 2022 14:14:45 +0100 Subject: [PATCH 2/4] remove test tag --- .gitlab-ci.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2a322d6632..c8bd082928 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -413,9 +413,6 @@ external-renderer-pytest-on-merge-request: extends: - .test-job-linux - .rules-merge-request - # TODO: remove, only for testing - tags: - - test-fhg-linux-runner1 needs: ["build-codec-linux-make"] # TODO: set reasonable timeout, will most likely take less timeout: "20 minutes" @@ -470,9 +467,6 @@ ivas-pytest-on-merge-request: extends: - .test-job-linux - .rules-merge-request - # TODO: remove, only for testing - tags: - - test-fhg-linux-runner1 stage: compare needs: ["build-codec-linux-cmake", "codec-smoke-test"] timeout: "10 minutes" @@ -522,9 +516,6 @@ evs-pytest-on-merge-request: extends: - .test-job-linux - .rules-merge-request - # TODO: remove, only for testing - tags: - - test-fhg-linux-runner1 stage: compare needs: ["build-codec-linux-cmake", "codec-smoke-test"] timeout: "10 minutes" @@ -646,9 +637,6 @@ codec-comparison-on-main-push: extends: - .test-job-linux - .rules-main-push - # TODO: remove, only for testing - tags: - - test-fhg-linux-runner1 stage: compare needs: ["build-codec-linux-cmake"] timeout: "30 minutes" # To be revisited -- GitLab From 3b3a98694198dba01a31cf5c2e82b741de007249 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 22 Nov 2022 14:20:03 +0100 Subject: [PATCH 3/4] use self-contained-html flag so report works standalone --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8800af5488..ef7291844c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -443,7 +443,7 @@ external-renderer-pytest-on-merge-request: # run test - exit_code=0 - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --junit-xml=report-junit.xml tests/renderer/test_renderer_be_comparison.py || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer_be_comparison.py || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check @@ -493,7 +493,7 @@ ivas-pytest-on-merge-request: ### run pytest - exit_code=0 - - python3 -m pytest tests -v --html=report.html --junit-xml=report-junit.xml || exit_code=$? + - python3 -m pytest tests -v --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check @@ -539,7 +539,7 @@ evs-pytest-on-merge-request: ### run pytest for EVS cases - exit_code=0 - - python3 -m pytest tests/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --html=report.html --junit-xml=report-junit-evs.xml || exit_code=$? + - python3 -m pytest tests/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --html=report.html --self-contained-html --junit-xml=report-junit-evs.xml || exit_code=$? - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check @@ -694,7 +694,7 @@ codec-comparison-on-main-push: ### run pytest - exit_code=0 - - python3 -m pytest tests -v --html=report.html --junit-xml=report-junit.xml || exit_code=$? + - python3 -m pytest tests -v --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code=$? - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi -- GitLab From 1e8c90b51ce8c6e71f625a71ab80c258d4c5f07a Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 22 Nov 2022 15:04:02 +0100 Subject: [PATCH 4/4] do not log env vars on runners --- pytest.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pytest.ini b/pytest.ini index bb4169677b..0cb53d22bb 100644 --- a/pytest.ini +++ b/pytest.ini @@ -12,3 +12,5 @@ junit_family = xunit1 log_file_level = DEBUG log_format = %(asctime)s %(levelname)s %(message)s log_date_format = %Y-%m-%d %H:%M:%S +# for pytest-html report: do not log environment variables from the runners +environment_table_redact_list = .* -- GitLab