From 245c6293ae71ef70deb60dbcd4fccc3ff75c9a7c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 17 Dec 2025 15:20:03 +0100 Subject: [PATCH 01/49] add first version of ci config for validation --- .gitlab-ci.yml | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..627e545 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,38 @@ +stages: + - test + +workflow: + rules: + # see https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines + - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + +variables: + REPO_URL_FLOAT: "https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec" + +check-merged-ci-config-is-valid: + stage: test + tags: + ivas-linux + variables: + MERGED_YAML: "merged.yaml" + REPO_URL: $REPO_URL_FLOAT + script: + - git clone -b main --single-branch $REPO_URL code + - cd code + - sed -i".bak" "s/\&IVAS_CODEC_CI_REF main/\&IVAS_CODEC_CI_REF ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}/" .gitlab-ci.yml + - | + jq --null-input --arg yaml "$(<.gitlab-ci.yml)" '.content=$yaml' \ + curl --url "https://forge.3gpp.org/rep/api/v4/projects/49/ci/lint?include_merged_yaml=true" \ + --header 'Content-Type: application/json' \ + --data @- + --output $MERGED_YAML + - | + grep "\"valid\": true" + artifacts: + when: always + access: all + expire_in: "5 days" + paths: + - $MERGED_YAML -- GitLab From 7d4f0b27ab6aa2580f2df5f27069adf85df1df3b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 17 Dec 2025 15:22:13 +0100 Subject: [PATCH 02/49] fix tags field --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 627e545..7446b7d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,7 +14,7 @@ variables: check-merged-ci-config-is-valid: stage: test tags: - ivas-linux + - ivas-linux variables: MERGED_YAML: "merged.yaml" REPO_URL: $REPO_URL_FLOAT -- GitLab From ddb21c824072c38010299765f42810101b28172b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 17 Dec 2025 15:37:30 +0100 Subject: [PATCH 03/49] parametrize job over repos --- .gitlab-ci.yml | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7446b7d..8099925 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,6 +10,9 @@ workflow: variables: REPO_URL_FLOAT: "https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec" + REPO_URL_BASOP: "https://forge.3gpp.org/rep/sa4/audio/ivas-basop" + PROJECT_ID_FLOAT: "49" + PROJECT_ID_BASOP: "77" check-merged-ci-config-is-valid: stage: test @@ -17,22 +20,38 @@ check-merged-ci-config-is-valid: - ivas-linux variables: MERGED_YAML: "merged.yaml" - REPO_URL: $REPO_URL_FLOAT + BASELINE_YAML: "baseline.yaml" + parallel: + matrix: + - REPO: + - ivas-codec + - ivas-basop script: - - git clone -b main --single-branch $REPO_URL code - - cd code - - sed -i".bak" "s/\&IVAS_CODEC_CI_REF main/\&IVAS_CODEC_CI_REF ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}/" .gitlab-ci.yml + - set -euxo pipefail + - if [ "$REPO" == "ivas-codec" ]; then repo_url=$REPO_URL_FLOAT; project_id=$PROJECT_ID_FLOAT; fi + - if [ "$REPO" == "ivas-basop" ]; then repo_url=$REPO_URL_BASOP; project_id=$PROJECT_ID_BASOP; fi + + - git clone -b main --single-branch $repo_url $REPO + - | + jq --null-input --arg yaml "$(<$REPO/.gitlab-ci.yml)" '.content=$yaml' \ + curl --url "https://forge.3gpp.org/rep/api/v4/projects/49/ci/lint?include_merged_yaml=true" \ + --header 'Content-Type: application/json' \ + --data @- + --output $BASELINE_YAML + + - sed -i".bak" "s/\&IVAS_CODEC_CI_REF main/\&IVAS_CODEC_CI_REF ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}/" $REPO/.gitlab-ci.yml - | - jq --null-input --arg yaml "$(<.gitlab-ci.yml)" '.content=$yaml' \ + jq --null-input --arg yaml "$(<$REPO/.gitlab-ci.yml)" '.content=$yaml' \ curl --url "https://forge.3gpp.org/rep/api/v4/projects/49/ci/lint?include_merged_yaml=true" \ --header 'Content-Type: application/json' \ --data @- --output $MERGED_YAML - | - grep "\"valid\": true" + grep "\"valid\": true" $MERGED_YAML artifacts: when: always access: all expire_in: "5 days" paths: - $MERGED_YAML + - $BASELINE_YAML -- GitLab From 222be4f4ff490ad809590f07711607e36d72774d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 17 Dec 2025 15:38:26 +0100 Subject: [PATCH 04/49] make interruptible --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8099925..58a0dff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,3 +1,7 @@ +default: + # by default, we want all jobs to be interruptible, and we need to configure this explicitly + interruptible: true + stages: - test -- GitLab From 5970c1f42748879b3d7a876dc223c59825eabdc3 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 17 Dec 2025 16:10:24 +0100 Subject: [PATCH 05/49] fix missing pipe --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 58a0dff..81a83eb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,7 +38,7 @@ check-merged-ci-config-is-valid: - git clone -b main --single-branch $repo_url $REPO - | jq --null-input --arg yaml "$(<$REPO/.gitlab-ci.yml)" '.content=$yaml' \ - curl --url "https://forge.3gpp.org/rep/api/v4/projects/49/ci/lint?include_merged_yaml=true" \ + | curl --url "https://forge.3gpp.org/rep/api/v4/projects/49/ci/lint?include_merged_yaml=true" \ --header 'Content-Type: application/json' \ --data @- --output $BASELINE_YAML @@ -46,7 +46,7 @@ check-merged-ci-config-is-valid: - sed -i".bak" "s/\&IVAS_CODEC_CI_REF main/\&IVAS_CODEC_CI_REF ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}/" $REPO/.gitlab-ci.yml - | jq --null-input --arg yaml "$(<$REPO/.gitlab-ci.yml)" '.content=$yaml' \ - curl --url "https://forge.3gpp.org/rep/api/v4/projects/49/ci/lint?include_merged_yaml=true" \ + | curl --url "https://forge.3gpp.org/rep/api/v4/projects/49/ci/lint?include_merged_yaml=true" \ --header 'Content-Type: application/json' \ --data @- --output $MERGED_YAML -- GitLab From d4b75f07ec2924b345d7cd436cc32f50a48ac045 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 17 Dec 2025 16:15:11 +0100 Subject: [PATCH 06/49] fix command linebreak --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 81a83eb..546757b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,16 +40,16 @@ check-merged-ci-config-is-valid: jq --null-input --arg yaml "$(<$REPO/.gitlab-ci.yml)" '.content=$yaml' \ | curl --url "https://forge.3gpp.org/rep/api/v4/projects/49/ci/lint?include_merged_yaml=true" \ --header 'Content-Type: application/json' \ + --output $BASELINE_YAML \ --data @- - --output $BASELINE_YAML - sed -i".bak" "s/\&IVAS_CODEC_CI_REF main/\&IVAS_CODEC_CI_REF ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}/" $REPO/.gitlab-ci.yml - | jq --null-input --arg yaml "$(<$REPO/.gitlab-ci.yml)" '.content=$yaml' \ | curl --url "https://forge.3gpp.org/rep/api/v4/projects/49/ci/lint?include_merged_yaml=true" \ --header 'Content-Type: application/json' \ + --output $MERGED_YAML \ --data @- - --output $MERGED_YAML - | grep "\"valid\": true" $MERGED_YAML artifacts: -- GitLab From 26c825e3658df4b2868e5fce97f0eee40877f117 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 17 Dec 2025 16:19:10 +0100 Subject: [PATCH 07/49] skip LFS files in cloning + fix curl URL with project id --- .gitlab-ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 546757b..12a4b78 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,10 +35,11 @@ check-merged-ci-config-is-valid: - if [ "$REPO" == "ivas-codec" ]; then repo_url=$REPO_URL_FLOAT; project_id=$PROJECT_ID_FLOAT; fi - if [ "$REPO" == "ivas-basop" ]; then repo_url=$REPO_URL_BASOP; project_id=$PROJECT_ID_BASOP; fi - - git clone -b main --single-branch $repo_url $REPO + # we only want the yaml file, skip LFS files for speedup + - GIT_LFS_SKIP_SMUDGE=1 git clone -b main --single-branch $repo_url $REPO - | jq --null-input --arg yaml "$(<$REPO/.gitlab-ci.yml)" '.content=$yaml' \ - | curl --url "https://forge.3gpp.org/rep/api/v4/projects/49/ci/lint?include_merged_yaml=true" \ + | curl --url "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" \ --header 'Content-Type: application/json' \ --output $BASELINE_YAML \ --data @- @@ -46,7 +47,7 @@ check-merged-ci-config-is-valid: - sed -i".bak" "s/\&IVAS_CODEC_CI_REF main/\&IVAS_CODEC_CI_REF ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}/" $REPO/.gitlab-ci.yml - | jq --null-input --arg yaml "$(<$REPO/.gitlab-ci.yml)" '.content=$yaml' \ - | curl --url "https://forge.3gpp.org/rep/api/v4/projects/49/ci/lint?include_merged_yaml=true" \ + | curl --url "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" \ --header 'Content-Type: application/json' \ --output $MERGED_YAML \ --data @- -- GitLab From eec0a3e5f437cb2a10e1f2d92c80d0f77f79a4c2 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 17 Dec 2025 16:53:20 +0100 Subject: [PATCH 08/49] add authentication via CI token --- .gitlab-ci.yml | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 12a4b78..2319a85 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,8 +23,8 @@ check-merged-ci-config-is-valid: tags: - ivas-linux variables: - MERGED_YAML: "merged.yaml" - BASELINE_YAML: "baseline.yaml" + MERGED_RESPONSE: "merged.json" + BASELINE_RESPONSE: "baseline.json" parallel: matrix: - REPO: @@ -38,25 +38,29 @@ check-merged-ci-config-is-valid: # we only want the yaml file, skip LFS files for speedup - GIT_LFS_SKIP_SMUDGE=1 git clone -b main --single-branch $repo_url $REPO - | - jq --null-input --arg yaml "$(<$REPO/.gitlab-ci.yml)" '.content=$yaml' \ - | curl --url "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" \ - --header 'Content-Type: application/json' \ - --output $BASELINE_YAML \ - --data @- + jq -n --arg yaml "$(<$REPO/.gitlab-ci.yml)" '.content=$yaml' \ + | curl -X POST \ + --url "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" \ + -H "Content-Type: application/json" \ + -H "JOB-TOKEN: $CI_JOB_TOKEN" \ + --output $BASELINE_RESPONSE \ + --data-binary @- - sed -i".bak" "s/\&IVAS_CODEC_CI_REF main/\&IVAS_CODEC_CI_REF ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}/" $REPO/.gitlab-ci.yml - | - jq --null-input --arg yaml "$(<$REPO/.gitlab-ci.yml)" '.content=$yaml' \ - | curl --url "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" \ - --header 'Content-Type: application/json' \ - --output $MERGED_YAML \ - --data @- + jq -n --arg yaml "$(<$REPO/.gitlab-ci.yml)" '.content=$yaml' \ + | curl -X POST \ + --url "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" \ + -H "Content-Type: application/json" \ + -H "JOB-TOKEN: $CI_JOB_TOKEN" \ + --output $MERGED_RESPONSE \ + --data-binary @- - | - grep "\"valid\": true" $MERGED_YAML + grep "\"valid\": true" $MERGED_RESPONSE artifacts: when: always access: all expire_in: "5 days" paths: - - $MERGED_YAML - - $BASELINE_YAML + - $MERGED_RESPONSE + - $BASELINE_RESPONSE -- GitLab From 0815628b5ba2bb6da3865e5e9a1b18ff9cdae9e9 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 17 Dec 2025 17:03:01 +0100 Subject: [PATCH 09/49] get ci yaml file directly without cloning --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2319a85..6c1ae51 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,10 +35,10 @@ check-merged-ci-config-is-valid: - if [ "$REPO" == "ivas-codec" ]; then repo_url=$REPO_URL_FLOAT; project_id=$PROJECT_ID_FLOAT; fi - if [ "$REPO" == "ivas-basop" ]; then repo_url=$REPO_URL_BASOP; project_id=$PROJECT_ID_BASOP; fi - # we only want the yaml file, skip LFS files for speedup - - GIT_LFS_SKIP_SMUDGE=1 git clone -b main --single-branch $repo_url $REPO + - test_yml="$REPO-ci.yml" + - curl --url "$repo_url/./raw/main/.gitlab-ci.yml?inline=false" --output $test_yml - | - jq -n --arg yaml "$(<$REPO/.gitlab-ci.yml)" '.content=$yaml' \ + jq -n --arg yaml "$(<$test_yml)" '.content=$yaml' \ | curl -X POST \ --url "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" \ -H "Content-Type: application/json" \ @@ -48,7 +48,7 @@ check-merged-ci-config-is-valid: - sed -i".bak" "s/\&IVAS_CODEC_CI_REF main/\&IVAS_CODEC_CI_REF ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}/" $REPO/.gitlab-ci.yml - | - jq -n --arg yaml "$(<$REPO/.gitlab-ci.yml)" '.content=$yaml' \ + jq -n --arg yaml "$(<$test_yml)" '.content=$yaml' \ | curl -X POST \ --url "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" \ -H "Content-Type: application/json" \ -- GitLab From b0d73e7cf705cbc1281a85ea0f985e3600107dd8 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 17 Dec 2025 17:04:14 +0100 Subject: [PATCH 10/49] fix path to file for sed command --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6c1ae51..8f33d6d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,7 +46,7 @@ check-merged-ci-config-is-valid: --output $BASELINE_RESPONSE \ --data-binary @- - - sed -i".bak" "s/\&IVAS_CODEC_CI_REF main/\&IVAS_CODEC_CI_REF ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}/" $REPO/.gitlab-ci.yml + - sed -i".bak" "s/\&IVAS_CODEC_CI_REF main/\&IVAS_CODEC_CI_REF ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}/" $test_yml - | jq -n --arg yaml "$(<$test_yml)" '.content=$yaml' \ | curl -X POST \ -- GitLab From 1236daeba8a02fe8ea638071ea882b9a7eab6ad5 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 17 Dec 2025 17:08:10 +0100 Subject: [PATCH 11/49] add dbg printout --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8f33d6d..39a5d27 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,6 +37,7 @@ check-merged-ci-config-is-valid: - test_yml="$REPO-ci.yml" - curl --url "$repo_url/./raw/main/.gitlab-ci.yml?inline=false" --output $test_yml + - cat $test_yml - | jq -n --arg yaml "$(<$test_yml)" '.content=$yaml' \ | curl -X POST \ -- GitLab From 5dd17874481d39dc6bd06d5c211023bedf8e931a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 17 Dec 2025 17:15:05 +0100 Subject: [PATCH 12/49] more dbg output --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 39a5d27..938a626 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,6 +35,12 @@ check-merged-ci-config-is-valid: - if [ "$REPO" == "ivas-codec" ]; then repo_url=$REPO_URL_FLOAT; project_id=$PROJECT_ID_FLOAT; fi - if [ "$REPO" == "ivas-basop" ]; then repo_url=$REPO_URL_BASOP; project_id=$PROJECT_ID_BASOP; fi + # for testing + - | + curl -s -o /dev/null -w "%{http_code}\n" -H "JOB-TOKEN: $CI_JOB_TOKEN" "https://forge.3gpp.org/rep/api/v4/projects/49" + - | + echo '{"content":"stages: [test]"}' | curl -s -o /dev/null -w "%{http_code}\n" -X POST -H "Content-Type: application/json" "https://forge.3gpp.org/rep/api/v4/projects/49/ci/lint" + - test_yml="$REPO-ci.yml" - curl --url "$repo_url/./raw/main/.gitlab-ci.yml?inline=false" --output $test_yml - cat $test_yml -- GitLab From 5bde5c5826634bac65b8e041dc81a2fad2469ddc Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 17 Dec 2025 17:18:00 +0100 Subject: [PATCH 13/49] add -v to curl --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 938a626..366e754 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,16 +37,16 @@ check-merged-ci-config-is-valid: # for testing - | - curl -s -o /dev/null -w "%{http_code}\n" -H "JOB-TOKEN: $CI_JOB_TOKEN" "https://forge.3gpp.org/rep/api/v4/projects/49" + curl -v -s -o /dev/null -w "%{http_code}\n" -H "JOB-TOKEN: $CI_JOB_TOKEN" "https://forge.3gpp.org/rep/api/v4/projects/49" - | echo '{"content":"stages: [test]"}' | curl -s -o /dev/null -w "%{http_code}\n" -X POST -H "Content-Type: application/json" "https://forge.3gpp.org/rep/api/v4/projects/49/ci/lint" - test_yml="$REPO-ci.yml" - - curl --url "$repo_url/./raw/main/.gitlab-ci.yml?inline=false" --output $test_yml + - curl -v --url "$repo_url/./raw/main/.gitlab-ci.yml?inline=false" --output $test_yml - cat $test_yml - | jq -n --arg yaml "$(<$test_yml)" '.content=$yaml' \ - | curl -X POST \ + | curl -v -X POST \ --url "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" \ -H "Content-Type: application/json" \ -H "JOB-TOKEN: $CI_JOB_TOKEN" \ -- GitLab From c6ab1b55f97e09c65f99335ffad5a923282708d4 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 17 Dec 2025 17:26:49 +0100 Subject: [PATCH 14/49] more debugging stuff --- .gitlab-ci.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 366e754..71adc74 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,11 +31,18 @@ check-merged-ci-config-is-valid: - ivas-codec - ivas-basop script: - - set -euxo pipefail + # - set -euxo pipefail - if [ "$REPO" == "ivas-codec" ]; then repo_url=$REPO_URL_FLOAT; project_id=$PROJECT_ID_FLOAT; fi - if [ "$REPO" == "ivas-basop" ]; then repo_url=$REPO_URL_BASOP; project_id=$PROJECT_ID_BASOP; fi # for testing + - | + echo '{"content":"stages: [test]"}' | curl -s -o /dev/null -w "%{http_code}\n" -X POST -H "Content-Type: application/json" -H "JOB-TOKEN: $CI_JOB_TOKEN" "https://forge.3gpp.org/rep/api/v4/projects/49/ci/lint" + - | + curl -s -H "JOB-TOKEN: $CI_JOB_TOKEN" "https://forge.3gpp.org/rep/api/v4/projects/ivas-codec-pc%2Fivas-codec-ci" | jq .id,.visibility + - | + curl -s -o /dev/null -w "%{http_code}\n" -H "JOB-TOKEN: $CI_JOB_TOKEN" "https://forge.3gpp.org/rep/api/v4/projects/ivas-codec-pc%2Fivas-codec-ci/repository/files/main-float.yml/raw?ref=main" + - | curl -v -s -o /dev/null -w "%{http_code}\n" -H "JOB-TOKEN: $CI_JOB_TOKEN" "https://forge.3gpp.org/rep/api/v4/projects/49" - | -- GitLab From e019937bd47de559c552fd2cfe069c0629414c21 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 17 Dec 2025 17:34:56 +0100 Subject: [PATCH 15/49] different debugging --- .gitlab-ci.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 71adc74..49fdb4c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,16 +37,11 @@ check-merged-ci-config-is-valid: # for testing - | - echo '{"content":"stages: [test]"}' | curl -s -o /dev/null -w "%{http_code}\n" -X POST -H "Content-Type: application/json" -H "JOB-TOKEN: $CI_JOB_TOKEN" "https://forge.3gpp.org/rep/api/v4/projects/49/ci/lint" + echo 'stages: [test]' | jq -Rs '{content: .}' | curl -sS -i -X POST -H "Content-Type: application/json" -H "JOB-TOKEN: $CI_JOB_TOKEN" --data-binary @- "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint" - | - curl -s -H "JOB-TOKEN: $CI_JOB_TOKEN" "https://forge.3gpp.org/rep/api/v4/projects/ivas-codec-pc%2Fivas-codec-ci" | jq .id,.visibility + jq -Rs '{content: .}' "$test_yml" | curl -sS -i -X POST -H "Content-Type: application/json" -H "JOB-TOKEN: $CI_JOB_TOKEN" --data-binary @- "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint" - | - curl -s -o /dev/null -w "%{http_code}\n" -H "JOB-TOKEN: $CI_JOB_TOKEN" "https://forge.3gpp.org/rep/api/v4/projects/ivas-codec-pc%2Fivas-codec-ci/repository/files/main-float.yml/raw?ref=main" - - - | - curl -v -s -o /dev/null -w "%{http_code}\n" -H "JOB-TOKEN: $CI_JOB_TOKEN" "https://forge.3gpp.org/rep/api/v4/projects/49" - - | - echo '{"content":"stages: [test]"}' | curl -s -o /dev/null -w "%{http_code}\n" -X POST -H "Content-Type: application/json" "https://forge.3gpp.org/rep/api/v4/projects/49/ci/lint" + jq -Rs '{content: .}' "$test_yml" | curl -sS -i -X POST -H "Content-Type: application/json" -H "JOB-TOKEN: $CI_JOB_TOKEN" --data-binary @- "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" - test_yml="$REPO-ci.yml" - curl -v --url "$repo_url/./raw/main/.gitlab-ci.yml?inline=false" --output $test_yml -- GitLab From 43f875011b744237cfdc8f68237136d250dcf6ea Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 17 Dec 2025 17:54:13 +0100 Subject: [PATCH 16/49] use new token --- .gitlab-ci.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 49fdb4c..21684be 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,11 +37,7 @@ check-merged-ci-config-is-valid: # for testing - | - echo 'stages: [test]' | jq -Rs '{content: .}' | curl -sS -i -X POST -H "Content-Type: application/json" -H "JOB-TOKEN: $CI_JOB_TOKEN" --data-binary @- "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint" - - | - jq -Rs '{content: .}' "$test_yml" | curl -sS -i -X POST -H "Content-Type: application/json" -H "JOB-TOKEN: $CI_JOB_TOKEN" --data-binary @- "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint" - - | - jq -Rs '{content: .}' "$test_yml" | curl -sS -i -X POST -H "Content-Type: application/json" -H "JOB-TOKEN: $CI_JOB_TOKEN" --data-binary @- "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" + echo 'stages: [test]' | jq -Rs '{content: .}' | curl -sS -i -X POST -H "Content-Type: application/json" -H "PRIVATE-TOKEN: $LINT_TOKEN" --data-binary @- "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint" - test_yml="$REPO-ci.yml" - curl -v --url "$repo_url/./raw/main/.gitlab-ci.yml?inline=false" --output $test_yml @@ -51,7 +47,7 @@ check-merged-ci-config-is-valid: | curl -v -X POST \ --url "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" \ -H "Content-Type: application/json" \ - -H "JOB-TOKEN: $CI_JOB_TOKEN" \ + -H "PRIVATE-TOKEN: $LINT_TOKEN" \ --output $BASELINE_RESPONSE \ --data-binary @- @@ -61,7 +57,7 @@ check-merged-ci-config-is-valid: | curl -X POST \ --url "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" \ -H "Content-Type: application/json" \ - -H "JOB-TOKEN: $CI_JOB_TOKEN" \ + -H "PRIVATE-TOKEN: $LINT_TOKEN" \ --output $MERGED_RESPONSE \ --data-binary @- - | -- GitLab From 9fd785a56756ef75f5d1fbf37cc1941b85c7fa64 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 17 Dec 2025 18:02:53 +0100 Subject: [PATCH 17/49] silent curl + different grep --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 21684be..83e61cf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,11 +40,11 @@ check-merged-ci-config-is-valid: echo 'stages: [test]' | jq -Rs '{content: .}' | curl -sS -i -X POST -H "Content-Type: application/json" -H "PRIVATE-TOKEN: $LINT_TOKEN" --data-binary @- "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint" - test_yml="$REPO-ci.yml" - - curl -v --url "$repo_url/./raw/main/.gitlab-ci.yml?inline=false" --output $test_yml + - curl -sS --url "$repo_url/./raw/main/.gitlab-ci.yml?inline=false" --output $test_yml - cat $test_yml - | jq -n --arg yaml "$(<$test_yml)" '.content=$yaml' \ - | curl -v -X POST \ + | curl -sS -X POST \ --url "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" \ -H "Content-Type: application/json" \ -H "PRIVATE-TOKEN: $LINT_TOKEN" \ @@ -54,14 +54,14 @@ check-merged-ci-config-is-valid: - sed -i".bak" "s/\&IVAS_CODEC_CI_REF main/\&IVAS_CODEC_CI_REF ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}/" $test_yml - | jq -n --arg yaml "$(<$test_yml)" '.content=$yaml' \ - | curl -X POST \ + | curl -sS POST \ --url "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" \ -H "Content-Type: application/json" \ -H "PRIVATE-TOKEN: $LINT_TOKEN" \ --output $MERGED_RESPONSE \ --data-binary @- - | - grep "\"valid\": true" $MERGED_RESPONSE + grep "valid" $MERGED_RESPONSE | grep "true" artifacts: when: always access: all -- GitLab From 607bb002ea181e1db8693d0c1551787ec2787db1 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 17 Dec 2025 18:05:20 +0100 Subject: [PATCH 18/49] fix typo --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 83e61cf..67154a8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -54,7 +54,7 @@ check-merged-ci-config-is-valid: - sed -i".bak" "s/\&IVAS_CODEC_CI_REF main/\&IVAS_CODEC_CI_REF ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}/" $test_yml - | jq -n --arg yaml "$(<$test_yml)" '.content=$yaml' \ - | curl -sS POST \ + | curl -sS -X POST \ --url "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" \ -H "Content-Type: application/json" \ -H "PRIVATE-TOKEN: $LINT_TOKEN" \ -- GitLab From 8bb6609a5ae58d9f5bfb10156fc03bd24f7b1259 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 17 Dec 2025 18:25:23 +0100 Subject: [PATCH 19/49] add comments + diffing --- .gitlab-ci.yml | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 67154a8..28b52cc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,6 +25,9 @@ check-merged-ci-config-is-valid: variables: MERGED_RESPONSE: "merged.json" BASELINE_RESPONSE: "baseline.json" + ARTIFACTS_MERGED_YAML: "merged.yml" + ARTIFACTS_BASELINE_YAML: "baseline.yml" + ARTIFACTS_DIFF_YAML: "diff.yml" parallel: matrix: - REPO: @@ -35,13 +38,12 @@ check-merged-ci-config-is-valid: - if [ "$REPO" == "ivas-codec" ]; then repo_url=$REPO_URL_FLOAT; project_id=$PROJECT_ID_FLOAT; fi - if [ "$REPO" == "ivas-basop" ]; then repo_url=$REPO_URL_BASOP; project_id=$PROJECT_ID_BASOP; fi - # for testing - - | - echo 'stages: [test]' | jq -Rs '{content: .}' | curl -sS -i -X POST -H "Content-Type: application/json" -H "PRIVATE-TOKEN: $LINT_TOKEN" --data-binary @- "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint" - + # get the CI file from the respective repo without lengthy clone - test_yml="$REPO-ci.yml" - curl -sS --url "$repo_url/./raw/main/.gitlab-ci.yml?inline=false" --output $test_yml - cat $test_yml + + # get the "baseline" yaml for comparison. this is always main - | jq -n --arg yaml "$(<$test_yml)" '.content=$yaml' \ | curl -sS -X POST \ @@ -51,6 +53,7 @@ check-merged-ci-config-is-valid: --output $BASELINE_RESPONSE \ --data-binary @- + # change branch to the one we are currently on and get the "merged" yaml - sed -i".bak" "s/\&IVAS_CODEC_CI_REF main/\&IVAS_CODEC_CI_REF ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}/" $test_yml - | jq -n --arg yaml "$(<$test_yml)" '.content=$yaml' \ @@ -60,6 +63,13 @@ check-merged-ci-config-is-valid: -H "PRIVATE-TOKEN: $LINT_TOKEN" \ --output $MERGED_RESPONSE \ --data-binary @- + + # extract actual merged yaml from the json responses and get diff + - cat $BASELINE_RESPONSE | jq -r ".merged_yaml" > $ARTIFACTS_BASELINE_YAML + - cat $MERGED_RESPONSE | jq -r ".merged_yaml" > $ARTIFACTS_MERGED_YAML + - dyff between $ARTIFACTS_BASELINE_YAML $ARTIFACTS_MERGED_YAML | tee $ARTIFACTS_DIFF_YAML + + # fail job if merged yaml is not valid - | grep "valid" $MERGED_RESPONSE | grep "true" artifacts: @@ -67,5 +77,6 @@ check-merged-ci-config-is-valid: access: all expire_in: "5 days" paths: - - $MERGED_RESPONSE - - $BASELINE_RESPONSE + - $ARTIFACTS_BASELINE_YAML + - $ARTIFACTS_MERGED_YAML + - $ARTIFACTS_DIFF_YAML -- GitLab From 6106f62c8a2286e333858504130b1ee81d3daee8 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 17 Dec 2025 19:03:13 +0100 Subject: [PATCH 20/49] use jq for validity check to avoid massive printout --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 28b52cc..11ea9bb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,8 +70,7 @@ check-merged-ci-config-is-valid: - dyff between $ARTIFACTS_BASELINE_YAML $ARTIFACTS_MERGED_YAML | tee $ARTIFACTS_DIFF_YAML # fail job if merged yaml is not valid - - | - grep "valid" $MERGED_RESPONSE | grep "true" + - if [ $(jq ".valid" $MERGED_RESPONSE) != "true" ]; then exit 1; fi artifacts: when: always access: all -- GitLab From 9fb4ec5585c08f800a6a73598f0c376aef255f25 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 17 Dec 2025 19:04:56 +0100 Subject: [PATCH 21/49] [revert-me] make invalid edit to test CI --- main-float.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/main-float.yml b/main-float.yml index 1cc0861..3013e5c 100644 --- a/main-float.yml +++ b/main-float.yml @@ -204,6 +204,7 @@ workflow: branch-is-up-to-date-with-main-pre: extends: - .rules-merge-request-to-main + - job-does-not-exist - .job-linux stage: prevalidate needs: [] -- GitLab From d7b01def0fe4ea19a2e427098b314086280dde66 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 17 Dec 2025 19:06:51 +0100 Subject: [PATCH 22/49] Revert "[revert-me] make invalid edit to test CI" This reverts commit 9fb4ec5585c08f800a6a73598f0c376aef255f25. --- main-float.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/main-float.yml b/main-float.yml index 3013e5c..1cc0861 100644 --- a/main-float.yml +++ b/main-float.yml @@ -204,7 +204,6 @@ workflow: branch-is-up-to-date-with-main-pre: extends: - .rules-merge-request-to-main - - job-does-not-exist - .job-linux stage: prevalidate needs: [] -- GitLab From fc0c6d2eb35ea8f2f983a656dbe158d77dd102de Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 17 Dec 2025 19:10:24 +0100 Subject: [PATCH 23/49] remove debug printout --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 11ea9bb..b1f7052 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,7 +41,6 @@ check-merged-ci-config-is-valid: # get the CI file from the respective repo without lengthy clone - test_yml="$REPO-ci.yml" - curl -sS --url "$repo_url/./raw/main/.gitlab-ci.yml?inline=false" --output $test_yml - - cat $test_yml # get the "baseline" yaml for comparison. this is always main - | -- GitLab From 73dfc558f22722b8fa76225782f3e52f55712ce0 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 10:00:31 +0100 Subject: [PATCH 24/49] try different token --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b1f7052..1bd1513 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -48,7 +48,7 @@ check-merged-ci-config-is-valid: | curl -sS -X POST \ --url "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" \ -H "Content-Type: application/json" \ - -H "PRIVATE-TOKEN: $LINT_TOKEN" \ + -H "PRIVATE-TOKEN: $LINT_TOKEN_FLOAT" \ --output $BASELINE_RESPONSE \ --data-binary @- @@ -59,7 +59,7 @@ check-merged-ci-config-is-valid: | curl -sS -X POST \ --url "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" \ -H "Content-Type: application/json" \ - -H "PRIVATE-TOKEN: $LINT_TOKEN" \ + -H "PRIVATE-TOKEN: $LINT_TOKEN_FLOAT" \ --output $MERGED_RESPONSE \ --data-binary @- -- GitLab From 0c13d130a9d8eb104ca54b7702e7bc55eb840a3d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 10:04:03 +0100 Subject: [PATCH 25/49] responses to artifacts --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1bd1513..9bb4fed 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -78,3 +78,5 @@ check-merged-ci-config-is-valid: - $ARTIFACTS_BASELINE_YAML - $ARTIFACTS_MERGED_YAML - $ARTIFACTS_DIFF_YAML + - $BASELINE_RESPONSE + - $MERGED_RESPONSE -- GitLab From 9fbe5f0d464f7426f959e4d56289fea6debeebaa Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 10:33:29 +0100 Subject: [PATCH 26/49] try CI_JOB_TOKEN again --- .gitlab-ci.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9bb4fed..90ff038 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,6 +38,17 @@ check-merged-ci-config-is-valid: - if [ "$REPO" == "ivas-codec" ]; then repo_url=$REPO_URL_FLOAT; project_id=$PROJECT_ID_FLOAT; fi - if [ "$REPO" == "ivas-basop" ]; then repo_url=$REPO_URL_BASOP; project_id=$PROJECT_ID_BASOP; fi + # for testing + - | + curl -H "JOB-TOKEN: $CI_JOB_TOKEN" "https://forge.3gpp.org/rep/api/v4/projects/$project_id/repository/files/.gitlab-ci.yml" + + - | + curl -X POST \ + -H "Content-Type: application/json" \ + -H "JOB-TOKEN: $CI_JOB_TOKEN" \ + --data '{"content": "stages: [test]\n"}' \ + "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" + # get the CI file from the respective repo without lengthy clone - test_yml="$REPO-ci.yml" - curl -sS --url "$repo_url/./raw/main/.gitlab-ci.yml?inline=false" --output $test_yml @@ -48,7 +59,8 @@ check-merged-ci-config-is-valid: | curl -sS -X POST \ --url "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" \ -H "Content-Type: application/json" \ - -H "PRIVATE-TOKEN: $LINT_TOKEN_FLOAT" \ + # -H "PRIVATE-TOKEN: $LINT_TOKEN_FLOAT" \ + -H "JOB-TOKEN: $CI_JOB_TOKEN" \ --output $BASELINE_RESPONSE \ --data-binary @- @@ -59,7 +71,8 @@ check-merged-ci-config-is-valid: | curl -sS -X POST \ --url "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" \ -H "Content-Type: application/json" \ - -H "PRIVATE-TOKEN: $LINT_TOKEN_FLOAT" \ + # -H "PRIVATE-TOKEN: $LINT_TOKEN_FLOAT" \ + -H "JOB-TOKEN: $CI_JOB_TOKEN" \ --output $MERGED_RESPONSE \ --data-binary @- -- GitLab From 3930eb34e916922c3a8f46a4e9e238c3e4019381 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 10:35:33 +0100 Subject: [PATCH 27/49] correct test commands --- .gitlab-ci.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 90ff038..cf25a65 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,14 +40,10 @@ check-merged-ci-config-is-valid: # for testing - | - curl -H "JOB-TOKEN: $CI_JOB_TOKEN" "https://forge.3gpp.org/rep/api/v4/projects/$project_id/repository/files/.gitlab-ci.yml" + curl -H "JOB-TOKEN: $CI_JOB_TOKEN" "https://forge.3gpp.org/rep/api/v4/projects/$project_id/repository/files/.gitlab-ci.yml?ref=main" - | - curl -X POST \ - -H "Content-Type: application/json" \ - -H "JOB-TOKEN: $CI_JOB_TOKEN" \ - --data '{"content": "stages: [test]\n"}' \ - "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" + curl -X POST -H "Content-Type: application/json" -H "JOB-TOKEN: $CI_JOB_TOKEN" --data '{"content": "stages: [test]\n"}' "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true&ref=main" # get the CI file from the respective repo without lengthy clone - test_yml="$REPO-ci.yml" -- GitLab From 4873af4135094de19925fd4eacbc91aac757c05e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 10:37:26 +0100 Subject: [PATCH 28/49] test prointout --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cf25a65..57f0311 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -43,7 +43,8 @@ check-merged-ci-config-is-valid: curl -H "JOB-TOKEN: $CI_JOB_TOKEN" "https://forge.3gpp.org/rep/api/v4/projects/$project_id/repository/files/.gitlab-ci.yml?ref=main" - | - curl -X POST -H "Content-Type: application/json" -H "JOB-TOKEN: $CI_JOB_TOKEN" --data '{"content": "stages: [test]\n"}' "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true&ref=main" + curl -X POST -H "Content-Type: application/json" -H "JOB-TOKEN: $CI_JOB_TOKEN" --data '{"content": "stages: [test]\n"}' "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true&ref=main" --output test_output.json + - cat test_output.json # get the CI file from the respective repo without lengthy clone - test_yml="$REPO-ci.yml" -- GitLab From 4370edde7b2d8b91232ed1a0f586d95241b97415 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 11:50:20 +0100 Subject: [PATCH 29/49] try with patching and running the lint locally --- .gitlab-ci.yml | 41 ++++++++++++++++------------------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 57f0311..ba69ebc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,12 +12,6 @@ workflow: when: never - if: $CI_PIPELINE_SOURCE == 'merge_request_event' -variables: - REPO_URL_FLOAT: "https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec" - REPO_URL_BASOP: "https://forge.3gpp.org/rep/sa4/audio/ivas-basop" - PROJECT_ID_FLOAT: "49" - PROJECT_ID_BASOP: "77" - check-merged-ci-config-is-valid: stage: test tags: @@ -28,6 +22,8 @@ check-merged-ci-config-is-valid: ARTIFACTS_MERGED_YAML: "merged.yml" ARTIFACTS_BASELINE_YAML: "baseline.yml" ARTIFACTS_DIFF_YAML: "diff.yml" + REPO_URL_FLOAT: "https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec" + REPO_URL_BASOP: "https://forge.3gpp.org/rep/sa4/audio/ivas-basop" parallel: matrix: - REPO: @@ -35,41 +31,36 @@ check-merged-ci-config-is-valid: - ivas-basop script: # - set -euxo pipefail - - if [ "$REPO" == "ivas-codec" ]; then repo_url=$REPO_URL_FLOAT; project_id=$PROJECT_ID_FLOAT; fi - - if [ "$REPO" == "ivas-basop" ]; then repo_url=$REPO_URL_BASOP; project_id=$PROJECT_ID_BASOP; fi - - # for testing - - | - curl -H "JOB-TOKEN: $CI_JOB_TOKEN" "https://forge.3gpp.org/rep/api/v4/projects/$project_id/repository/files/.gitlab-ci.yml?ref=main" - - - | - curl -X POST -H "Content-Type: application/json" -H "JOB-TOKEN: $CI_JOB_TOKEN" --data '{"content": "stages: [test]\n"}' "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true&ref=main" --output test_output.json - - cat test_output.json + - if [ "$REPO" == "ivas-codec" ]; then repo_url=$REPO_URL_FLOAT; patch_yaml="main-float.yml"; fi + - if [ "$REPO" == "ivas-basop" ]; then repo_url=$REPO_URL_BASOP; patch_yaml="main-basop-yml";fi # get the CI file from the respective repo without lengthy clone - test_yml="$REPO-ci.yml" - curl -sS --url "$repo_url/./raw/main/.gitlab-ci.yml?inline=false" --output $test_yml + - cat $test_yml - # get the "baseline" yaml for comparison. this is always main + # get the "baseline" yaml for comparison. this is always the thing we got from main branch of $REPO - | jq -n --arg yaml "$(<$test_yml)" '.content=$yaml' \ | curl -sS -X POST \ - --url "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" \ + --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true" \ -H "Content-Type: application/json" \ - # -H "PRIVATE-TOKEN: $LINT_TOKEN_FLOAT" \ - -H "JOB-TOKEN: $CI_JOB_TOKEN" \ --output $BASELINE_RESPONSE \ --data-binary @- - # change branch to the one we are currently on and get the "merged" yaml - - sed -i".bak" "s/\&IVAS_CODEC_CI_REF main/\&IVAS_CODEC_CI_REF ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}/" $test_yml + # patch the yaml file to include the stuff we have locally in this branch + - | + sed -i".bak" -E '/^include:/,/^[^[:space:]]/c\ (base) + include:\ + - local: ${patch_yaml}' $test_yml + - cat $test_yml + + # lint patched yaml - | jq -n --arg yaml "$(<$test_yml)" '.content=$yaml' \ | curl -sS -X POST \ - --url "https://forge.3gpp.org/rep/api/v4/projects/$project_id/ci/lint?include_merged_yaml=true" \ + --url "${CI_API_V4_URL}/v4/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true" \ -H "Content-Type: application/json" \ - # -H "PRIVATE-TOKEN: $LINT_TOKEN_FLOAT" \ - -H "JOB-TOKEN: $CI_JOB_TOKEN" \ --output $MERGED_RESPONSE \ --data-binary @- -- GitLab From aae8dd10fe938b614afd39b95b49b68b102cd02c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 11:53:39 +0100 Subject: [PATCH 30/49] add job token to curl call --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ba69ebc..96bc240 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -45,6 +45,7 @@ check-merged-ci-config-is-valid: | curl -sS -X POST \ --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true" \ -H "Content-Type: application/json" \ + -H "JOB-TOKEN: $CI_JOB_TOKEN" \ --output $BASELINE_RESPONSE \ --data-binary @- @@ -61,6 +62,7 @@ check-merged-ci-config-is-valid: | curl -sS -X POST \ --url "${CI_API_V4_URL}/v4/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true" \ -H "Content-Type: application/json" \ + -H "JOB-TOKEN: $CI_JOB_TOKEN" \ --output $MERGED_RESPONSE \ --data-binary @- -- GitLab From b96006b959ab14be96fc9b701d97b4f298f67fcb Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 11:54:56 +0100 Subject: [PATCH 31/49] turn on bash strcit mode --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 96bc240..812fccc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,7 +30,7 @@ check-merged-ci-config-is-valid: - ivas-codec - ivas-basop script: - # - set -euxo pipefail + - set -euxo pipefail - if [ "$REPO" == "ivas-codec" ]; then repo_url=$REPO_URL_FLOAT; patch_yaml="main-float.yml"; fi - if [ "$REPO" == "ivas-basop" ]; then repo_url=$REPO_URL_BASOP; patch_yaml="main-basop-yml";fi -- GitLab From 834ba5d930ae9776f13d803b3e7fc87adeee4dc0 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 13:01:59 +0100 Subject: [PATCH 32/49] different patching --- .gitlab-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 812fccc..3631ee3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,8 +31,8 @@ check-merged-ci-config-is-valid: - ivas-basop script: - set -euxo pipefail - - if [ "$REPO" == "ivas-codec" ]; then repo_url=$REPO_URL_FLOAT; patch_yaml="main-float.yml"; fi - - if [ "$REPO" == "ivas-basop" ]; then repo_url=$REPO_URL_BASOP; patch_yaml="main-basop-yml";fi + - if [ "$REPO" == "ivas-codec" ]; then repo_url=$REPO_URL_FLOAT; patch_yml="main-float.yml"; fi + - if [ "$REPO" == "ivas-basop" ]; then repo_url=$REPO_URL_BASOP; patch_yml="main-basop-yml";fi # get the CI file from the respective repo without lengthy clone - test_yml="$REPO-ci.yml" @@ -51,9 +51,9 @@ check-merged-ci-config-is-valid: # patch the yaml file to include the stuff we have locally in this branch - | - sed -i".bak" -E '/^include:/,/^[^[:space:]]/c\ (base) - include:\ - - local: ${patch_yaml}' $test_yml + sed '/^include:/,$d' $test_yml > $test_yml + - | + echo -e "include:\n - local: ${patch_yml}\n" >> $test_yml - cat $test_yml # lint patched yaml -- GitLab From 791695842696560876606686204fe92092d9b251 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 13:04:15 +0100 Subject: [PATCH 33/49] add cat --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3631ee3..9299c38 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -52,6 +52,7 @@ check-merged-ci-config-is-valid: # patch the yaml file to include the stuff we have locally in this branch - | sed '/^include:/,$d' $test_yml > $test_yml + - cat $test_yml - | echo -e "include:\n - local: ${patch_yml}\n" >> $test_yml - cat $test_yml -- GitLab From ea3acb6b17f98971748984b8b1734c49632d6917 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 13:06:08 +0100 Subject: [PATCH 34/49] inplace sed --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9299c38..3438ae1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -51,7 +51,7 @@ check-merged-ci-config-is-valid: # patch the yaml file to include the stuff we have locally in this branch - | - sed '/^include:/,$d' $test_yml > $test_yml + sed -i".bak" '/^include:/,$d' $test_yml - cat $test_yml - | echo -e "include:\n - local: ${patch_yml}\n" >> $test_yml -- GitLab From eef655bd291e77b44efcc747f389ccb14e666f43 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 13:07:32 +0100 Subject: [PATCH 35/49] fix typo in URL --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3438ae1..e44cfce 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -61,7 +61,7 @@ check-merged-ci-config-is-valid: - | jq -n --arg yaml "$(<$test_yml)" '.content=$yaml' \ | curl -sS -X POST \ - --url "${CI_API_V4_URL}/v4/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true" \ + --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true" \ -H "Content-Type: application/json" \ -H "JOB-TOKEN: $CI_JOB_TOKEN" \ --output $MERGED_RESPONSE \ -- GitLab From 2058ff32dcd1dded12b482453f265ad48c81dfc6 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 13:19:00 +0100 Subject: [PATCH 36/49] project access token for this project now --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e44cfce..9046c2f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -59,11 +59,11 @@ check-merged-ci-config-is-valid: # lint patched yaml - | - jq -n --arg yaml "$(<$test_yml)" '.content=$yaml' \ + jq -n --arg yaml "$(<$test_yml)" --arg ref "$CI_COMMIT_REF_NAME" '{ content: $yaml, ref: $ref }' \ | curl -sS -X POST \ --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true" \ -H "Content-Type: application/json" \ - -H "JOB-TOKEN: $CI_JOB_TOKEN" \ + -H "PRIVATE-TOKEN: $LINT_TOKEN" \ --output $MERGED_RESPONSE \ --data-binary @- -- GitLab From 2ce856d05fdca6738c3e181c2640dcb690a177a9 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 13:25:10 +0100 Subject: [PATCH 37/49] change name of variable --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9046c2f..d05c7a3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -63,7 +63,7 @@ check-merged-ci-config-is-valid: | curl -sS -X POST \ --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true" \ -H "Content-Type: application/json" \ - -H "PRIVATE-TOKEN: $LINT_TOKEN" \ + -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \ --output $MERGED_RESPONSE \ --data-binary @- -- GitLab From fc020dcd4c154b1f3e8da558a182e0fd7d1eb3f5 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 13:31:10 +0100 Subject: [PATCH 38/49] add getting the baseline yml --- .gitlab-ci.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d05c7a3..7758412 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,13 +39,24 @@ check-merged-ci-config-is-valid: - curl -sS --url "$repo_url/./raw/main/.gitlab-ci.yml?inline=false" --output $test_yml - cat $test_yml - # get the "baseline" yaml for comparison. this is always the thing we got from main branch of $REPO + # get the "baseline" yaml for comparison. this is always the thing we have on the MR target branch (usually main) + - git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME + - base_yml="base-$CI_MERGE_REQUEST_TARGET_BRANCH_NAME-ci.yml + # patch the yaml file to include the stuff we have locally in this branch + - cp $test_yml $base_yml + - | + sed -i".bak" '/^include:/,$d' $base_yml + - cat $base_yml - | - jq -n --arg yaml "$(<$test_yml)" '.content=$yaml' \ + echo -e "include:\n - local: ${patch_yml}\n" >> $base_yml + - cat $base_yml + # lint patched yaml + - | + jq -n --arg yaml "$(<$base_yml)" --arg ref "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" '{ content: $yaml, ref: $ref }' \ | curl -sS -X POST \ --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true" \ -H "Content-Type: application/json" \ - -H "JOB-TOKEN: $CI_JOB_TOKEN" \ + -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \ --output $BASELINE_RESPONSE \ --data-binary @- -- GitLab From a9b275bcf6c0618b00c4590f0523bd3d3cc378fc Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 13:35:32 +0100 Subject: [PATCH 39/49] fix missing "" --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7758412..fed8710 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,7 +41,7 @@ check-merged-ci-config-is-valid: # get the "baseline" yaml for comparison. this is always the thing we have on the MR target branch (usually main) - git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME - - base_yml="base-$CI_MERGE_REQUEST_TARGET_BRANCH_NAME-ci.yml + - base_yml="base-$CI_MERGE_REQUEST_TARGET_BRANCH_NAME-ci.yml" # patch the yaml file to include the stuff we have locally in this branch - cp $test_yml $base_yml - | @@ -60,6 +60,7 @@ check-merged-ci-config-is-valid: --output $BASELINE_RESPONSE \ --data-binary @- + - git checkout $CI_COMMIT_REF_NAME # patch the yaml file to include the stuff we have locally in this branch - | sed -i".bak" '/^include:/,$d' $test_yml -- GitLab From 05b6d283a5f9db88ae34518c6f52769679a7a8d8 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 13:37:48 +0100 Subject: [PATCH 40/49] [revert-me] add change for testing CI --- main-float.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/main-float.yml b/main-float.yml index 1cc0861..20908b3 100644 --- a/main-float.yml +++ b/main-float.yml @@ -21,6 +21,7 @@ stages: - compare - postvalidate - deploy + - new variables: EVS_BE_TEST_DIR: "/usr/local/be_2_evs_test" -- GitLab From 4285df3fd30f0129e69bec1bd235e8ce1e00b1b2 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 13:41:34 +0100 Subject: [PATCH 41/49] [revert-me] new var for testing --- main-float.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/main-float.yml b/main-float.yml index 20908b3..e60dd18 100644 --- a/main-float.yml +++ b/main-float.yml @@ -24,6 +24,7 @@ stages: - new variables: + NEW_VARIABLE_FOR_TESTIN: "vlavsdsfsdf" EVS_BE_TEST_DIR: "/usr/local/be_2_evs_test" EVS_BE_WIN_TEST_DIR: "C:/Users/gitlab-runner/testvec" PROCESSING_SCRIPTS_BIN_DIR: "/test-bin" -- GitLab From f776cd8b52380725c879cd9aa02c2bdcc870520b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 13:48:20 +0100 Subject: [PATCH 42/49] pass ref as part of URL --- .gitlab-ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fed8710..5afebee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,6 +41,7 @@ check-merged-ci-config-is-valid: # get the "baseline" yaml for comparison. this is always the thing we have on the MR target branch (usually main) - git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME + - target_sha=$(git rev-parse HEAD) - base_yml="base-$CI_MERGE_REQUEST_TARGET_BRANCH_NAME-ci.yml" # patch the yaml file to include the stuff we have locally in this branch - cp $test_yml $base_yml @@ -52,9 +53,9 @@ check-merged-ci-config-is-valid: - cat $base_yml # lint patched yaml - | - jq -n --arg yaml "$(<$base_yml)" --arg ref "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" '{ content: $yaml, ref: $ref }' \ + jq -n --arg yaml "$(<$base_yml)" '{ content: $yaml }' \ | curl -sS -X POST \ - --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true" \ + --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&ref=${target_sha}" \ -H "Content-Type: application/json" \ -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \ --output $BASELINE_RESPONSE \ @@ -71,9 +72,9 @@ check-merged-ci-config-is-valid: # lint patched yaml - | - jq -n --arg yaml "$(<$test_yml)" --arg ref "$CI_COMMIT_REF_NAME" '{ content: $yaml, ref: $ref }' \ + jq -n --arg yaml "$(<$test_yml)" '{ content: $yaml }' \ | curl -sS -X POST \ - --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true" \ + --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&ref=${CI_COMMIT_SHA}" \ -H "Content-Type: application/json" \ -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \ --output $MERGED_RESPONSE \ -- GitLab From b30cdd6784b6924be59432611e94e3393811a301 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 13:58:40 +0100 Subject: [PATCH 43/49] print yml file head --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5afebee..d08efb6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -51,6 +51,7 @@ check-merged-ci-config-is-valid: - | echo -e "include:\n - local: ${patch_yml}\n" >> $base_yml - cat $base_yml + - head -n 50 $patch_yml # lint patched yaml - | jq -n --arg yaml "$(<$base_yml)" '{ content: $yaml }' \ @@ -69,6 +70,7 @@ check-merged-ci-config-is-valid: - | echo -e "include:\n - local: ${patch_yml}\n" >> $test_yml - cat $test_yml + - head -n 50 $patch_yml # lint patched yaml - | -- GitLab From 0d2834e5b359b08cf9facd9853d3a579204d1515 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 17:17:18 +0100 Subject: [PATCH 44/49] rework + more verbose debug output --- .gitlab-ci.yml | 136 +++++++++++++++++++++++++++++++------------------ 1 file changed, 87 insertions(+), 49 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d08efb6..e090a9a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,4 @@ default: - # by default, we want all jobs to be interruptible, and we need to configure this explicitly interruptible: true stages: @@ -7,7 +6,6 @@ stages: workflow: rules: - # see https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" when: never - if: $CI_PIPELINE_SOURCE == 'merge_request_event' @@ -16,12 +14,8 @@ check-merged-ci-config-is-valid: stage: test tags: - ivas-linux + variables: - MERGED_RESPONSE: "merged.json" - BASELINE_RESPONSE: "baseline.json" - ARTIFACTS_MERGED_YAML: "merged.yml" - ARTIFACTS_BASELINE_YAML: "baseline.yml" - ARTIFACTS_DIFF_YAML: "diff.yml" REPO_URL_FLOAT: "https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec" REPO_URL_BASOP: "https://forge.3gpp.org/rep/sa4/audio/ivas-basop" parallel: @@ -29,73 +23,117 @@ check-merged-ci-config-is-valid: - REPO: - ivas-codec - ivas-basop + script: - set -euxo pipefail - - if [ "$REPO" == "ivas-codec" ]; then repo_url=$REPO_URL_FLOAT; patch_yml="main-float.yml"; fi - - if [ "$REPO" == "ivas-basop" ]; then repo_url=$REPO_URL_BASOP; patch_yml="main-basop-yml";fi - - # get the CI file from the respective repo without lengthy clone - - test_yml="$REPO-ci.yml" - - curl -sS --url "$repo_url/./raw/main/.gitlab-ci.yml?inline=false" --output $test_yml - - cat $test_yml - - # get the "baseline" yaml for comparison. this is always the thing we have on the MR target branch (usually main) - - git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME - - target_sha=$(git rev-parse HEAD) - - base_yml="base-$CI_MERGE_REQUEST_TARGET_BRANCH_NAME-ci.yml" - # patch the yaml file to include the stuff we have locally in this branch - - cp $test_yml $base_yml + - | - sed -i".bak" '/^include:/,$d' $base_yml - - cat $base_yml + + if [ "$REPO" == "ivas-codec" ]; then + repo_url=$REPO_URL_FLOAT + patch_yml="main-float.yml" + else + repo_url=$REPO_URL_BASOP + patch_yml="main-basop.yml" + fi + + # artifact names + - MERGED_RESPONSE="${REPO}-merged.json" + - BASELINE_RESPONSE="${REPO}-baseline.json" + - ARTIFACTS_MERGED_YAML="${REPO}-merged.yml" + - ARTIFACTS_BASELINE_YAML="${REPO}-baseline.yml" + - ARTIFACTS_DIFF_YAML="${REPO}-diff.yml" + - TEST_YML="${REPO}-ci.yml" + - BASE_YML="${REPO}-base-ci.yml" + + # Get CI file from the respective repo + - curl -sS --url "${repo_url}/-/raw/main/.gitlab-ci.yml?inline=false" --output $TEST_YML + - echo "=== Original CI file ===" + - cat $TEST_YML + - | - echo -e "include:\n - local: ${patch_yml}\n" >> $base_yml - - cat $base_yml - - head -n 50 $patch_yml - # lint patched yaml + echo "Target branch SHA: $CI_MERGE_REQUEST_TARGET_BRANCH_SHA" + - | + echo "Current commit SHA: $CI_COMMIT_SHA" + + - git fetch origin + - git checkout $CI_COMMIT_REF_NAME + + - cp $TEST_YML $BASE_YML + - sed -i '/^include:/,$d' $BASE_YML + - | + echo -e "include:\n - local: ${patch_yml}" >> $BASE_YML + - echo "=== Baseline CI file (target branch) ===" + - cat $BASE_YML + + # Lint baseline with target branch ref - | - jq -n --arg yaml "$(<$base_yml)" '{ content: $yaml }' \ + jq -n --arg yaml "$(<$BASE_YML)" '{ content: $yaml }' \ | curl -sS -X POST \ - --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&ref=${target_sha}" \ + --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&ref=${CI_MERGE_REQUEST_TARGET_BRANCH_SHA}" \ -H "Content-Type: application/json" \ -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \ --output $BASELINE_RESPONSE \ --data-binary @- + - cat $BASELINE_RESPONSE | jq - - git checkout $CI_COMMIT_REF_NAME - # patch the yaml file to include the stuff we have locally in this branch - - | - sed -i".bak" '/^include:/,$d' $test_yml - - cat $test_yml + - sed -i '/^include:/,$d' $TEST_YML - | - echo -e "include:\n - local: ${patch_yml}\n" >> $test_yml - - cat $test_yml + echo -e "include:\n - local: ${patch_yml}" >> $TEST_YML + - echo "=== Current CI file (feature branch) ===" + - cat $TEST_YML + + # Show what the local file looks like + - echo "=== First 50 lines of ${patch_yml} ===" - head -n 50 $patch_yml - # lint patched yaml + # Lint current with feature branch ref - | - jq -n --arg yaml "$(<$test_yml)" '{ content: $yaml }' \ + jq -n --arg yaml "$(<$TEST_YML)" '{ content: $yaml }' \ | curl -sS -X POST \ --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&ref=${CI_COMMIT_SHA}" \ -H "Content-Type: application/json" \ -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \ --output $MERGED_RESPONSE \ --data-binary @- + - cat $MERGED_RESPONSE | jq + + - jq -r ".merged_yaml" $BASELINE_RESPONSE > $ARTIFACTS_BASELINE_YAML + - jq -r ".merged_yaml" $MERGED_RESPONSE > $ARTIFACTS_MERGED_YAML + - echo "=== Computing diff ===" + - dyff between $ARTIFACTS_BASELINE_YAML $ARTIFACTS_MERGED_YAML | tee $ARTIFACTS_DIFF_YAML || true + + # Check validity + - baseline_valid=$(jq -r ".valid" $BASELINE_RESPONSE) + - merged_valid=$(jq -r ".valid" $MERGED_RESPONSE) + - | + echo "Baseline valid: $baseline_valid" + - | + echo "Merged valid: $merged_valid" + + # Show errors if invalid + - | + + if [ "$merged_valid" != "true" ]; then + echo "=== MERGED CONFIG IS INVALID ===" + jq -r ".errors" $MERGED_RESPONSE + exit 1 + fi + + - | - # extract actual merged yaml from the json responses and get diff - - cat $BASELINE_RESPONSE | jq -r ".merged_yaml" > $ARTIFACTS_BASELINE_YAML - - cat $MERGED_RESPONSE | jq -r ".merged_yaml" > $ARTIFACTS_MERGED_YAML - - dyff between $ARTIFACTS_BASELINE_YAML $ARTIFACTS_MERGED_YAML | tee $ARTIFACTS_DIFF_YAML + if [ "$baseline_valid" != "true" ]; then + echo "=== WARNING: BASELINE CONFIG IS INVALID ===" + jq -r ".errors" $BASELINE_RESPONSE + fi - # fail job if merged yaml is not valid - - if [ $(jq ".valid" $MERGED_RESPONSE) != "true" ]; then exit 1; fi artifacts: when: always access: all expire_in: "5 days" paths: - - $ARTIFACTS_BASELINE_YAML - - $ARTIFACTS_MERGED_YAML - - $ARTIFACTS_DIFF_YAML - - $BASELINE_RESPONSE - - $MERGED_RESPONSE + - "*-baseline.yml" + - "*-merged.yml" + - "*-diff.yml" + - "*-baseline.json" + - "*-merged.json" -- GitLab From dea1cceb7bea2be27db2b3bb3b98d2257aae0c7b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 17:19:29 +0100 Subject: [PATCH 45/49] even more debug output!! --- .gitlab-ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e090a9a..6fb38bf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -66,6 +66,13 @@ check-merged-ci-config-is-valid: - echo "=== Baseline CI file (target branch) ===" - cat $BASE_YML + # Debug: Check if files actually differ between refs + - echo "=== Checking ${patch_yml} at both refs ===" + - | + curl -sS "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/repository/files/${patch_yml}?ref=${CI_MERGE_REQUEST_TARGET_BRANCH_SHA}" -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" | jq -r .content | base64 -d | sha256sum + - | + curl -sS "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/repository/files/${patch_yml}?ref=${CI_COMMIT_SHA}" -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" | jq -r .content | base64 -d | sha256sum + # Lint baseline with target branch ref - | jq -n --arg yaml "$(<$BASE_YML)" '{ content: $yaml }' \ -- GitLab From 33113c92c5be602f2574d82801b40e8ab3ee770c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 17:29:03 +0100 Subject: [PATCH 46/49] set dry_run=true --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6fb38bf..53cb3bb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -77,7 +77,7 @@ check-merged-ci-config-is-valid: - | jq -n --arg yaml "$(<$BASE_YML)" '{ content: $yaml }' \ | curl -sS -X POST \ - --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&ref=${CI_MERGE_REQUEST_TARGET_BRANCH_SHA}" \ + --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&dry_run=true&ref=${CI_MERGE_REQUEST_TARGET_BRANCH_SHA}" \ -H "Content-Type: application/json" \ -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \ --output $BASELINE_RESPONSE \ @@ -98,7 +98,7 @@ check-merged-ci-config-is-valid: - | jq -n --arg yaml "$(<$TEST_YML)" '{ content: $yaml }' \ | curl -sS -X POST \ - --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&ref=${CI_COMMIT_SHA}" \ + --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&dry_run=true&ref=${CI_COMMIT_SHA}" \ -H "Content-Type: application/json" \ -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \ --output $MERGED_RESPONSE \ -- GitLab From 67bb81c3cd95e7b652c51c1851eaedf20aad7d5c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 17:31:39 +0100 Subject: [PATCH 47/49] pass branch names for ref --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 53cb3bb..3addf90 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -77,7 +77,7 @@ check-merged-ci-config-is-valid: - | jq -n --arg yaml "$(<$BASE_YML)" '{ content: $yaml }' \ | curl -sS -X POST \ - --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&dry_run=true&ref=${CI_MERGE_REQUEST_TARGET_BRANCH_SHA}" \ + --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&dry_run=true&ref=${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" \ -H "Content-Type: application/json" \ -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \ --output $BASELINE_RESPONSE \ @@ -98,7 +98,7 @@ check-merged-ci-config-is-valid: - | jq -n --arg yaml "$(<$TEST_YML)" '{ content: $yaml }' \ | curl -sS -X POST \ - --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&dry_run=true&ref=${CI_COMMIT_SHA}" \ + --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&dry_run=true&ref=${CI_COMMIT_REF_NAME}" \ -H "Content-Type: application/json" \ -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \ --output $MERGED_RESPONSE \ -- GitLab From 895b64e19ea8ed18e40150fd3c683f29e7b79bb2 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 17:45:22 +0100 Subject: [PATCH 48/49] do extra lint step --- .gitlab-ci.yml | 66 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3addf90..fa74b26 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -73,16 +73,27 @@ check-merged-ci-config-is-valid: - | curl -sS "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/repository/files/${patch_yml}?ref=${CI_COMMIT_SHA}" -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" | jq -r .content | base64 -d | sha256sum - # Lint baseline with target branch ref + # Get merged baseline with target branch ref - | - jq -n --arg yaml "$(<$BASE_YML)" '{ content: $yaml }' \ + jq -n --arg yaml "$(<$BASE_YML)" '{ content: $yaml, dry_run: true }' \ | curl -sS -X POST \ - --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&dry_run=true&ref=${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" \ - -H "Content-Type: application/json" \ - -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \ - --output $BASELINE_RESPONSE \ - --data-binary @- - - cat $BASELINE_RESPONSE | jq + --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&dry_run=true&ref=${CI_MERGE_REQUEST_TARGET_BRANCH_SHA}" \ + -H "Content-Type: application/json" \ + -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \ + --output $BASELINE_RESPONSE \ + --data-binary @- + + # lint the whole merged thing statically (previous call tries to run pipeline which does not work because ivas-codec does not have push pipelines...) + - jq -r ".merged_yaml" $BASELINE_RESPONSE > $ARTIFACTS_BASELINE_YAML + - rm $BASELINE_RESPONSE + - | + jq -n --arg yaml "$(<$ARTIFACTS_BASELINE_YAML)" '{ content: $yaml }' \ + | curl -sS -X POST \ + --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?" \ + -H "Content-Type: application/json" \ + -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \ + --output $BASELINE_RESPONSE \ + --data-binary @- - sed -i '/^include:/,$d' $TEST_YML - | @@ -94,19 +105,28 @@ check-merged-ci-config-is-valid: - echo "=== First 50 lines of ${patch_yml} ===" - head -n 50 $patch_yml - # Lint current with feature branch ref + # Get current with feature branch ref - | - jq -n --arg yaml "$(<$TEST_YML)" '{ content: $yaml }' \ + jq -n --arg yaml "$(<$TEST_YML)" '{ content: $yaml, dry_run: true }' \ | curl -sS -X POST \ - --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&dry_run=true&ref=${CI_COMMIT_REF_NAME}" \ - -H "Content-Type: application/json" \ - -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \ - --output $MERGED_RESPONSE \ - --data-binary @- - - cat $MERGED_RESPONSE | jq + --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&dry_run=true&ref=${CI_COMMIT_SHA}" \ + -H "Content-Type: application/json" \ + -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \ + --output $MERGED_RESPONSE \ + --data-binary @- - - jq -r ".merged_yaml" $BASELINE_RESPONSE > $ARTIFACTS_BASELINE_YAML + # lint the whole merged thing statically (previous call tries to run pipeline which does not work because ivas-codec does not have push pipelines...) - jq -r ".merged_yaml" $MERGED_RESPONSE > $ARTIFACTS_MERGED_YAML + - rm $MERGED_RESPONSE + - | + jq -n --arg yaml "$(<$ARTIFACTS_MERGED_YAML)" '{ content: $yaml }' \ + | curl -sS -X POST \ + --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?" \ + -H "Content-Type: application/json" \ + -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \ + --output $MERGED_RESPONSE \ + --data-binary @- + - echo "=== Computing diff ===" - dyff between $ARTIFACTS_BASELINE_YAML $ARTIFACTS_MERGED_YAML | tee $ARTIFACTS_DIFF_YAML || true @@ -120,19 +140,17 @@ check-merged-ci-config-is-valid: # Show errors if invalid - | - + if [ "$baseline_valid" != "true" ]; then + echo "=== WARNING: BASELINE CONFIG IS INVALID ===" + jq -r ".errors" $BASELINE_RESPONSE + fi + - | if [ "$merged_valid" != "true" ]; then echo "=== MERGED CONFIG IS INVALID ===" jq -r ".errors" $MERGED_RESPONSE exit 1 fi - - | - - if [ "$baseline_valid" != "true" ]; then - echo "=== WARNING: BASELINE CONFIG IS INVALID ===" - jq -r ".errors" $BASELINE_RESPONSE - fi artifacts: when: always -- GitLab From 399f7b1f5364e84a1e9cf1aab5411c95bb99dc9e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 18 Dec 2025 17:49:26 +0100 Subject: [PATCH 49/49] add cat --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fa74b26..d313c5f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -85,6 +85,7 @@ check-merged-ci-config-is-valid: # lint the whole merged thing statically (previous call tries to run pipeline which does not work because ivas-codec does not have push pipelines...) - jq -r ".merged_yaml" $BASELINE_RESPONSE > $ARTIFACTS_BASELINE_YAML + - cat $ARTIFACTS_BASELINE_YAML - rm $BASELINE_RESPONSE - | jq -n --arg yaml "$(<$ARTIFACTS_BASELINE_YAML)" '{ content: $yaml }' \ @@ -118,6 +119,7 @@ check-merged-ci-config-is-valid: # lint the whole merged thing statically (previous call tries to run pipeline which does not work because ivas-codec does not have push pipelines...) - jq -r ".merged_yaml" $MERGED_RESPONSE > $ARTIFACTS_MERGED_YAML - rm $MERGED_RESPONSE + - cat $ARTIFACTS_MERGED_YAML - | jq -n --arg yaml "$(<$ARTIFACTS_MERGED_YAML)" '{ content: $yaml }' \ | curl -sS -X POST \ -- GitLab