From eae986c5dec8a9efe419f1310745513d52b2206a Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 31 Aug 2023 14:26:15 +0200 Subject: [PATCH 1/6] remove obsolete job and move voip test from compare to test stage --- .gitlab-ci.yml | 43 ++----------------------------------------- 1 file changed, 2 insertions(+), 41 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a84756e3aa..17987d8c9b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -770,8 +770,8 @@ voip-be-on-merge-request: extends: - .test-job-linux-needs-testv-dir - .rules-merge-request - stage: compare # Or should it be test? Comparison is done within one git revision - needs: ["build-codec-linux-make", codec-smoke-test] + stage: test + needs: ["build-codec-linux-make", "codec-smoke-test"] timeout: "10 minutes" script: - *print-common-info @@ -857,45 +857,6 @@ check-first-frame-is-sid: expose_as: "logs-sidstart" expire_in: "5 days" -selection-test-processing: - extends: - - .test-job-linux-needs-testv-dir - - .rules-merge-request - tags: - - processing-scripts-linux - stage: test - needs: ["build-codec-linux-make"] - script: - # get processing script code - - git clone https://forge.3gpp.org/rep/ivas-codec-pc/ivas-processing-scripts.git --single-branch -b main - - - mkdir -p ivas-processing-scripts/tests/data/testv - - cp -r scripts/testv/* ivas-processing-scripts/tests/data/testv/ - - # copy binaries into local bin dir, those should take precendence over PATH - - cp $PROCESSING_SCRIPTS_BIN_DIR/* ivas-processing-scripts/ivas_processing_scripts/bin/ - # for testing with native binaries - - rm ivas-processing-scripts/ivas_processing_scripts/bin/IVAS*.exe - - rm ivas-processing-scripts/ivas_processing_scripts/bin/EVS*.exe - - # build codec and put into bin dir - - make -j - - cp ./IVAS_* ivas-processing-scripts/ivas_processing_scripts/bin/ - - # patch the use_windows_codec_binaries key (weird folding is needed so colons are accepted) - - > - sed -i "s/use_windows_codec_binaries: true/use_windows_codec_binaries: false/" ivas-processing-scripts/experiments/selection/*/config/*.yml - - # run experiments test - - cd ivas-processing-scripts - - python3 -m pytest tests/test_experiments.py::test_generate_test_items -n auto | tee log.txt - artifacts: - paths: - - ivas-processing-scripts/experiments/selection/*/proc_output/*.log - - log.txt - when: on_failure - expire_in: 1 week - lc3plus-ensure-no-code-changes: extends: - .test-job-linux -- GitLab From 0040f99225ab13b67d39876745856393a18e0111 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 31 Aug 2023 15:21:08 +0200 Subject: [PATCH 2/6] restructure validation jobs --- .gitlab-ci.yml | 42 ++++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 17987d8c9b..05b872e3ac 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,10 +38,11 @@ workflow: stages: - .pre - maintenance + - prevalidate - build - test - compare - - validate + - postvalidate - deploy # --------------------------------------------------------------- @@ -133,6 +134,14 @@ stages: - sed -i.bak -e "s/\/\*[[:space:]]*\(#define[[:space:]]*SPLIT_REND_WITH_HEAD_ROT\)[[:space:]]*\*\//\1/g" ./lib_com/options.h - sed -i.bak -e "s/\/\/[[:space:]]*\(#define[[:space:]]*SPLIT_REND_WITH_HEAD_ROT\)/\1/g" ./lib_com/options.h +.get-commits-behind-count: &get-commits-behind-count + - echo $CI_COMMIT_SHA + - echo $CI_MERGE_REQUEST_TARGET_BRANCH_NAME + - commits_behind_count=$(git rev-list --count $CI_COMMIT_SHA..origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME) + +.check-commits-behind-count-in-compare-jobs: &check-commits-behind-count-in-compare-jobs + - if [ $commits_behind_count -ne 0 ]; then echo "Your branch is not up-to-date with main -> Compare tests will not run as they can contain false negatives this way.\nMain might have changed during your pipeline run. Run `git pull origin/main` to update."; exit 1; fi; + # --------------------------------------------------------------- # Job templates # --------------------------------------------------------------- @@ -179,11 +188,13 @@ stages: .build-job-linux: stage: build timeout: "4 minutes" + needs: [] tags: - ivas-linux .build-job-windows: stage: build + needs: [] timeout: "4 minutes" tags: - ivas-windows @@ -236,19 +247,28 @@ uninterruptible: # Validation jobs # --------------------------------------------------------------- -check-if-branch-is-up-to-date-with-main: +branch-is-up-to-date-with-main-pre: extends: - .rules-merge-request - stage: validate + stage: prevalidate needs: [] tags: - ivas-linux script: - - echo $CI_COMMIT_SHA - - echo $CI_MERGE_REQUEST_TARGET_BRANCH_NAME - - commits_behind_count=$(git rev-list --count $CI_COMMIT_SHA..origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME) + - *get-commits-behind-count + - echo $commits_behind_count + - if [ $commits_behind_count -eq 0 ]; then exit 0; else echo "Your branch is behind main, run `git pull origin/main` to update."; exit 1; fi; + +branch-is-up-to-date-with-main-post: + extends: + - .rules-merge-request + stage: postvalidate + tags: + - ivas-linux + script: + - *get-commits-behind-count - echo $commits_behind_count - - if [ $commits_behind_count -eq 0 ]; then exit 0; else exit 1; fi; + - if [ $commits_behind_count -eq 0 ]; then exit 0; else echo "Your branch is behind main, possibly main changed during your pipeline run, run `git pull origin/main` to update." exit 1; fi; # --------------------------------------------------------------- # Build jobs @@ -530,6 +550,8 @@ renderer-pytest-on-merge-request: stage: compare script: - *print-common-info + - *get-commits-behind-count + - # some helper variables - "|| true" to prevent failures from grep not finding anything # write to temporary file as workaround for failures observed with piping echo @@ -783,7 +805,7 @@ clang-format-check: - .rules-merge-request variables: ARTIFACT_BASE_NAME: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--formatting-fix" - stage: validate + stage: prevalidate needs: [] timeout: "5 minutes" script: @@ -830,7 +852,7 @@ check-first-frame-is-sid: tags: - ivas-linux stage: test - # needs: ["build-codec-linux-cmake"] + needs: ["build-codec-linux-cmake"] script: - *print-common-info - *update-ltv-repo @@ -861,7 +883,7 @@ lc3plus-ensure-no-code-changes: extends: - .test-job-linux - .rules-merge-request - stage: validate + stage: postvalidate needs: [] timeout: "5 minutes" script: -- GitLab From df2e94bc45e4179dc4a7ad2d0b3b39ad620be69e Mon Sep 17 00:00:00 2001 From: kiene Date: Thu, 31 Aug 2023 13:26:46 +0000 Subject: [PATCH 3/6] fix yaml error and use new anchor everywhere it is supposed to be used --- .gitlab-ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 05b872e3ac..e35e12e6eb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -551,7 +551,7 @@ renderer-pytest-on-merge-request: script: - *print-common-info - *get-commits-behind-count - - + - *check-commits-behind-count-in-compare-jobs # some helper variables - "|| true" to prevent failures from grep not finding anything # write to temporary file as workaround for failures observed with piping echo @@ -633,6 +633,7 @@ split-rendering-pytest-on-merge-request: stage: compare script: - *print-common-info + - *check-commits-behind-count-in-compare-jobs # some helper variables - "|| true" to prevent failures from grep not finding anything # write to temporary file as workaround for failures observed with piping echo @@ -696,6 +697,7 @@ ivas-pytest-on-merge-request: timeout: "10 minutes" script: - *print-common-info + - *check-commits-behind-count-in-compare-jobs - *merge-request-comparison-setup-codec # some helper variables - "|| true" to prevent failures from grep not finding anything @@ -748,6 +750,7 @@ evs-pytest-on-merge-request: timeout: "10 minutes" script: - *print-common-info + - *check-commits-behind-count-in-compare-jobs - *merge-request-comparison-setup-codec # some helper variables - "|| true" to prevent failures from grep not finding anything -- GitLab From afb8ed03b2b4077aeaaabdbe4e6b81d662c73f94 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 1 Sep 2023 09:06:00 +0200 Subject: [PATCH 4/6] fix printout and if statement --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e35e12e6eb..9ef34b55f0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -140,7 +140,7 @@ stages: - commits_behind_count=$(git rev-list --count $CI_COMMIT_SHA..origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME) .check-commits-behind-count-in-compare-jobs: &check-commits-behind-count-in-compare-jobs - - if [ $commits_behind_count -ne 0 ]; then echo "Your branch is not up-to-date with main -> Compare tests will not run as they can contain false negatives this way.\nMain might have changed during your pipeline run. Run `git pull origin/main` to update."; exit 1; fi; + - if [ $commits_behind_count -ne 0 ]; then echo "Your branch is not up-to-date with main -> Compare tests will not run as they can contain false negatives this way.\nMain might have changed during your pipeline run. Run `git merge origin/main` to update."; exit 1; fi # --------------------------------------------------------------- # Job templates @@ -257,7 +257,7 @@ branch-is-up-to-date-with-main-pre: script: - *get-commits-behind-count - echo $commits_behind_count - - if [ $commits_behind_count -eq 0 ]; then exit 0; else echo "Your branch is behind main, run `git pull origin/main` to update."; exit 1; fi; + - if [ $commits_behind_count -eq 0 ]; then exit 0; else echo "Your branch is behind main, run `git merge origin/main` to update."; exit 1; fi; branch-is-up-to-date-with-main-post: extends: @@ -268,7 +268,7 @@ branch-is-up-to-date-with-main-post: script: - *get-commits-behind-count - echo $commits_behind_count - - if [ $commits_behind_count -eq 0 ]; then exit 0; else echo "Your branch is behind main, possibly main changed during your pipeline run, run `git pull origin/main` to update." exit 1; fi; + - if [ $commits_behind_count -eq 0 ]; then exit 0; else echo "Your branch is behind main, possibly main changed during your pipeline run, run `git merge origin/main` to update." exit 1; fi; # --------------------------------------------------------------- # Build jobs -- GitLab From 5d5dcf269acc9ed6162c0a7bda9b739758fbb001 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 1 Sep 2023 09:47:15 +0200 Subject: [PATCH 5/6] fix command unintentionally running in printout and add anchor calls --- .gitlab-ci.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9ef34b55f0..9038fae597 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -140,7 +140,7 @@ stages: - commits_behind_count=$(git rev-list --count $CI_COMMIT_SHA..origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME) .check-commits-behind-count-in-compare-jobs: &check-commits-behind-count-in-compare-jobs - - if [ $commits_behind_count -ne 0 ]; then echo "Your branch is not up-to-date with main -> Compare tests will not run as they can contain false negatives this way.\nMain might have changed during your pipeline run. Run `git merge origin/main` to update."; exit 1; fi + - if [ $commits_behind_count -ne 0 ]; then echo "Your branch is not up-to-date with main -> Compare tests will not run as they can contain false negatives this way.\nMain might have changed during your pipeline run. Run 'git merge origin/main' to update."; exit 1; fi # --------------------------------------------------------------- # Job templates @@ -257,7 +257,7 @@ branch-is-up-to-date-with-main-pre: script: - *get-commits-behind-count - echo $commits_behind_count - - if [ $commits_behind_count -eq 0 ]; then exit 0; else echo "Your branch is behind main, run `git merge origin/main` to update."; exit 1; fi; + - if [ $commits_behind_count -eq 0 ]; then exit 0; else echo "Your branch is behind main, run 'git merge origin/main' to update."; exit 1; fi; branch-is-up-to-date-with-main-post: extends: @@ -268,7 +268,7 @@ branch-is-up-to-date-with-main-post: script: - *get-commits-behind-count - echo $commits_behind_count - - if [ $commits_behind_count -eq 0 ]; then exit 0; else echo "Your branch is behind main, possibly main changed during your pipeline run, run `git merge origin/main` to update." exit 1; fi; + - if [ $commits_behind_count -eq 0 ]; then exit 0; else echo "Your branch is behind main, possibly main changed during your pipeline run, run 'git merge origin/main' to update." exit 1; fi; # --------------------------------------------------------------- # Build jobs @@ -633,6 +633,7 @@ split-rendering-pytest-on-merge-request: stage: compare script: - *print-common-info + - *get-commits-behind-count - *check-commits-behind-count-in-compare-jobs # some helper variables - "|| true" to prevent failures from grep not finding anything @@ -697,6 +698,7 @@ ivas-pytest-on-merge-request: timeout: "10 minutes" script: - *print-common-info + - *get-commits-behind-count - *check-commits-behind-count-in-compare-jobs - *merge-request-comparison-setup-codec @@ -750,6 +752,7 @@ evs-pytest-on-merge-request: timeout: "10 minutes" script: - *print-common-info + - *get-commits-behind-count - *check-commits-behind-count-in-compare-jobs - *merge-request-comparison-setup-codec -- GitLab From 6455d7372eb77c951fe8146cf8939ba8feb27687 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Fri, 1 Sep 2023 15:23:39 +0200 Subject: [PATCH 6/6] remove upload-selection-BE-log job - remove upload-selection-BE-log job as it is no longer needed. --- .gitlab-ci.yml | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9038fae597..5d8ba91262 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1511,27 +1511,6 @@ complexity-StereoDmxEVS-stereo-in-mono-out: # Other jobs # --------------------------------------------------------------- -upload-selection-BE-log: - rules: - - if: $UPLOAD_SELECTION_BE_RESULTS && $CI_PIPELINE_SOURCE == 'trigger' - when: always - timeout: 5 minutes - tags: - - ericsson-windows-runner - script: - - cp -r $SELECTION_BE_RESULT ./selection-BE-result - - Get-Content -Path selection-BE-result/public_log--sha-*.txt - - $has_failed_debugging_active = (Select-String -Path selection-BE-result/public_log--sha-*.txt -Pattern '^FAILED tests' -CaseSensitive).Line - - Get-Content -Path selection-BE-result/public_log_DEBUGGING_inactive--sha-*.txt - - $has_failed_debugging_inactive = (Select-String -Path selection-BE-result/public_log_DEBUGGING_inactive--sha-*.txt -Pattern '^FAILED tests' -CaseSensitive).Line - - If($has_failed_debugging_active -or $has_failed_debugging_inactive) {exit -1} - artifacts: - paths: - - selection-BE-result/public_log--sha-*.txt - - selection-BE-result/public_log_DEBUGGING_inactive--sha-*.txt - when: always - expire_in: 1 week - # job that sets up gitlab pages website pages: stage: deploy -- GitLab