From 2a7fe14acd4b4da3623797166b81aa35d6704508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 23 Apr 2024 15:02:23 +0200 Subject: [PATCH 1/6] Use a type alias for a CI job --- src/ci/github-actions/calculate-job-matrix.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py index c24cefa8d89..a54081f83f1 100755 --- a/src/ci/github-actions/calculate-job-matrix.py +++ b/src/ci/github-actions/calculate-job-matrix.py @@ -19,8 +19,10 @@ import yaml JOBS_YAML_PATH = Path(__file__).absolute().parent / "jobs.yml" +Job = Dict[str, Any] -def name_jobs(jobs: List[Dict], prefix: str) -> List[Dict]: + +def name_jobs(jobs: List[Dict], prefix: str) -> List[Job]: """ Add a `name` attribute to each job, based on its image and the given `prefix`. """ @@ -29,7 +31,7 @@ def name_jobs(jobs: List[Dict], prefix: str) -> List[Dict]: return jobs -def add_base_env(jobs: List[Dict], environment: Dict[str, str]) -> List[Dict]: +def add_base_env(jobs: List[Job], environment: Dict[str, str]) -> List[Job]: """ Prepends `environment` to the `env` attribute of each job. The `env` of each job has higher precedence than `environment`. @@ -77,7 +79,7 @@ def find_job_type(ctx: GitHubCtx) -> Optional[JobType]: return None -def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Dict[str, Any]]: +def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Job]: if job_type == JobType.PR: return add_base_env(name_jobs(job_data["pr"], "PR"), job_data["envs"]["pr"]) elif job_type == JobType.Try: From e25735908fc4ee84011252bedfedf51e1de22b95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 23 Apr 2024 15:06:18 +0200 Subject: [PATCH 2/6] Skip jobs based on the active channel in Python --- src/ci/github-actions/calculate-job-matrix.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py index a54081f83f1..84c4ee40668 100755 --- a/src/ci/github-actions/calculate-job-matrix.py +++ b/src/ci/github-actions/calculate-job-matrix.py @@ -17,6 +17,7 @@ from typing import List, Dict, Any, Optional import yaml +CI_DIR = Path(__file__).absolute().parent.parent JOBS_YAML_PATH = Path(__file__).absolute().parent / "jobs.yml" Job = Dict[str, Any] @@ -90,6 +91,13 @@ def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Job]: return [] +def skip_jobs(jobs: List[Dict[str, Any]], channel: str) -> List[Job]: + """ + Skip CI jobs that are not supposed to be executed on the given `channel`. + """ + return [j for j in jobs if j.get("CI_ONLY_WHEN_CHANNEL", channel) == channel] + + def get_github_ctx() -> GitHubCtx: return GitHubCtx( event_name=os.environ["GITHUB_EVENT_NAME"], @@ -109,9 +117,13 @@ if __name__ == "__main__": job_type = find_job_type(github_ctx) logging.info(f"Job type: {job_type}") + with open(CI_DIR / "channel") as f: + channel = f.read().strip() + jobs = [] if job_type is not None: jobs = calculate_jobs(job_type, data) + jobs = skip_jobs(jobs, channel) logging.info(f"Output:\n{yaml.dump(jobs, indent=4)}") print(f"jobs={json.dumps(jobs)}") From 246ee5363710a6cc505400f02addc5db56ff0977 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 23 Apr 2024 15:06:44 +0200 Subject: [PATCH 3/6] Remove `should-skip-this.sh` --- .github/workflows/ci.yml | 3 --- src/ci/github-actions/ci.yml | 4 ---- src/ci/scripts/should-skip-this.sh | 21 --------------------- 3 files changed, 28 deletions(-) delete mode 100755 src/ci/scripts/should-skip-this.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 872b671b031..149640140d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,9 +94,6 @@ jobs: env: EXTRA_VARIABLES: "${{ toJson(matrix.env) }}" if: success() && !env.SKIP_JOB - - name: decide whether to skip this job - run: src/ci/scripts/should-skip-this.sh - if: success() && !env.SKIP_JOB - name: ensure the channel matches the target branch run: src/ci/scripts/verify-channel.sh if: success() && !env.SKIP_JOB diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 19d6b517552..9947bb44ed4 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -163,10 +163,6 @@ x--expand-yaml-anchors--remove: EXTRA_VARIABLES: ${{ toJson(matrix.env) }} <<: *step - - name: decide whether to skip this job - run: src/ci/scripts/should-skip-this.sh - <<: *step - - name: ensure the channel matches the target branch run: src/ci/scripts/verify-channel.sh <<: *step diff --git a/src/ci/scripts/should-skip-this.sh b/src/ci/scripts/should-skip-this.sh deleted file mode 100755 index 48127166ad0..00000000000 --- a/src/ci/scripts/should-skip-this.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# Set the SKIP_JOB environment variable if this job is not supposed to run on the current builder. - -set -euo pipefail -IFS=$'\n\t' - -source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" - -if [[ -n "${CI_ONLY_WHEN_CHANNEL-}" ]]; then - if [[ "${CI_ONLY_WHEN_CHANNEL}" = "$(cat src/ci/channel)" ]]; then - echo "The channel is the expected one" - else - echo "Not executing this job as the channel is not the expected one" - ciCommandSetEnv SKIP_JOB 1 - exit 0 - fi -fi - - -echo "Executing the job since there is no skip rule preventing the execution" -exit 0 From 4942a35f24b1e97c8ca119a2bada670fbba6e477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 23 Apr 2024 15:09:22 +0200 Subject: [PATCH 4/6] Remove step YAML anchor and `env.SKIP_JOB` --- .github/workflows/ci.yml | 28 ++-------------------------- src/ci/github-actions/ci.yml | 32 ++------------------------------ 2 files changed, 4 insertions(+), 56 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 149640140d9..3651ef2c614 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,79 +88,57 @@ jobs: run: "echo \"[CI_PR_NUMBER=$num]\"" env: num: "${{ github.event.number }}" - if: "success() && !env.SKIP_JOB && github.event_name == 'pull_request'" + if: "success() && github.event_name == 'pull_request'" - name: add extra environment variables run: src/ci/scripts/setup-environment.sh env: EXTRA_VARIABLES: "${{ toJson(matrix.env) }}" - if: success() && !env.SKIP_JOB - name: ensure the channel matches the target branch run: src/ci/scripts/verify-channel.sh - if: success() && !env.SKIP_JOB - name: collect CPU statistics run: src/ci/scripts/collect-cpu-stats.sh - if: success() && !env.SKIP_JOB - name: show the current environment run: src/ci/scripts/dump-environment.sh - if: success() && !env.SKIP_JOB - name: install awscli run: src/ci/scripts/install-awscli.sh - if: success() && !env.SKIP_JOB - name: install sccache run: src/ci/scripts/install-sccache.sh - if: success() && !env.SKIP_JOB - name: select Xcode run: src/ci/scripts/select-xcode.sh - if: success() && !env.SKIP_JOB - name: install clang run: src/ci/scripts/install-clang.sh - if: success() && !env.SKIP_JOB - name: install tidy run: src/ci/scripts/install-tidy.sh - if: success() && !env.SKIP_JOB - name: install WIX run: src/ci/scripts/install-wix.sh - if: success() && !env.SKIP_JOB - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - if: success() && !env.SKIP_JOB - name: checkout submodules run: src/ci/scripts/checkout-submodules.sh - if: success() && !env.SKIP_JOB - name: install MSYS2 run: src/ci/scripts/install-msys2.sh - if: success() && !env.SKIP_JOB - name: install MinGW run: src/ci/scripts/install-mingw.sh - if: success() && !env.SKIP_JOB - name: install ninja run: src/ci/scripts/install-ninja.sh - if: success() && !env.SKIP_JOB - name: enable ipv6 on Docker run: src/ci/scripts/enable-docker-ipv6.sh - if: success() && !env.SKIP_JOB - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - if: success() && !env.SKIP_JOB - name: ensure line endings are correct run: src/ci/scripts/verify-line-endings.sh - if: success() && !env.SKIP_JOB - name: ensure backported commits are in upstream branches run: src/ci/scripts/verify-backported-commits.sh - if: success() && !env.SKIP_JOB - name: ensure the stable version number is correct run: src/ci/scripts/verify-stable-version-number.sh - if: success() && !env.SKIP_JOB - name: run the build run: src/ci/scripts/run-build-from-ci.sh 2>&1 env: AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" - if: success() && !env.SKIP_JOB - name: create github artifacts run: src/ci/scripts/create-doc-artifacts.sh - if: success() && !env.SKIP_JOB - name: upload artifacts to github uses: actions/upload-artifact@v4 with: @@ -168,13 +146,12 @@ jobs: path: obj/artifacts/doc if-no-files-found: ignore retention-days: 5 - if: success() && !env.SKIP_JOB - name: upload artifacts to S3 run: src/ci/scripts/upload-artifacts.sh env: AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" - if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')" + if: "success() && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')" master: name: master runs-on: ubuntu-latest @@ -199,7 +176,6 @@ jobs: shell: bash env: TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" - if: success() && !env.SKIP_JOB try-success: needs: - job diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 9947bb44ed4..bc4b1b815cf 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -107,9 +107,6 @@ x--expand-yaml-anchors--remove: - &job-aarch64-linux os: [self-hosted, ARM64, linux] - - &step - if: success() && !env.SKIP_JOB - - &base-ci-job defaults: run: @@ -151,7 +148,7 @@ x--expand-yaml-anchors--remove: run: echo "[CI_PR_NUMBER=$num]" env: num: ${{ github.event.number }} - if: success() && !env.SKIP_JOB && github.event_name == 'pull_request' + if: success() && github.event_name == 'pull_request' - name: add extra environment variables run: src/ci/scripts/setup-environment.sh @@ -161,67 +158,51 @@ x--expand-yaml-anchors--remove: # are passed to the `setup-environment.sh` script encoded in JSON, # which then uses log commands to actually set them. EXTRA_VARIABLES: ${{ toJson(matrix.env) }} - <<: *step - name: ensure the channel matches the target branch run: src/ci/scripts/verify-channel.sh - <<: *step - name: collect CPU statistics run: src/ci/scripts/collect-cpu-stats.sh - <<: *step - name: show the current environment run: src/ci/scripts/dump-environment.sh - <<: *step - name: install awscli run: src/ci/scripts/install-awscli.sh - <<: *step - name: install sccache run: src/ci/scripts/install-sccache.sh - <<: *step - name: select Xcode run: src/ci/scripts/select-xcode.sh - <<: *step - name: install clang run: src/ci/scripts/install-clang.sh - <<: *step - name: install tidy run: src/ci/scripts/install-tidy.sh - <<: *step - name: install WIX run: src/ci/scripts/install-wix.sh - <<: *step - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - <<: *step - name: checkout submodules run: src/ci/scripts/checkout-submodules.sh - <<: *step - name: install MSYS2 run: src/ci/scripts/install-msys2.sh - <<: *step - name: install MinGW run: src/ci/scripts/install-mingw.sh - <<: *step - name: install ninja run: src/ci/scripts/install-ninja.sh - <<: *step - name: enable ipv6 on Docker run: src/ci/scripts/enable-docker-ipv6.sh - <<: *step # Disable automatic line ending conversion (again). On Windows, when we're # installing dependencies, something switches the git configuration directory or @@ -230,19 +211,15 @@ x--expand-yaml-anchors--remove: # appropriate line endings. - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - <<: *step - name: ensure line endings are correct run: src/ci/scripts/verify-line-endings.sh - <<: *step - name: ensure backported commits are in upstream branches run: src/ci/scripts/verify-backported-commits.sh - <<: *step - name: ensure the stable version number is correct run: src/ci/scripts/verify-stable-version-number.sh - <<: *step - name: run the build # Redirect stderr to stdout to avoid reordering the two streams in the GHA logs. @@ -251,11 +228,9 @@ x--expand-yaml-anchors--remove: AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }} TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }} - <<: *step - name: create github artifacts run: src/ci/scripts/create-doc-artifacts.sh - <<: *step - name: upload artifacts to github uses: actions/upload-artifact@v4 @@ -265,7 +240,6 @@ x--expand-yaml-anchors--remove: path: obj/artifacts/doc if-no-files-found: ignore retention-days: 5 - <<: *step - name: upload artifacts to S3 run: src/ci/scripts/upload-artifacts.sh @@ -277,8 +251,7 @@ x--expand-yaml-anchors--remove: # adding the condition is helpful as this way CI will not silently skip # deploying artifacts from a dist builder if the variables are misconfigured, # erroring about invalid credentials instead. - if: success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1') - <<: *step + if: success() && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1') # These snippets are used by the try-success, try-failure, auto-success and auto-failure jobs. # Check out their documentation for more information on why they're needed. @@ -395,7 +368,6 @@ jobs: shell: bash env: TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }} - <<: *step # These jobs don't actually test anything, but they're used to tell bors the # build completed, as there is no practical way to detect when a workflow is From c59bc9a06ab0de13d96f02616b9476c5426fe54c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 24 Apr 2024 12:15:17 +0200 Subject: [PATCH 5/6] Fix documentation --- src/ci/github-actions/jobs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index ec58bd0924e..0f5dfaa5785 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -79,7 +79,7 @@ pr: <<: *job-linux-16c # Jobs that run when you perform a try build (@bors try) -# These jobs automatically inherit envs.production, to avoid repeating +# These jobs automatically inherit envs.try, to avoid repeating # it in each job definition. try: - image: dist-x86_64-linux @@ -88,7 +88,7 @@ try: <<: *job-linux-16c # Main CI jobs that have to be green to merge a commit into master -# These jobs automatically inherit envs.production, to avoid repeating +# These jobs automatically inherit envs.auto, to avoid repeating # it in each job definition. auto: ############################# From c36d78ed64d005046c0e57e28dee314ae64cb0ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 24 Apr 2024 12:32:25 +0200 Subject: [PATCH 6/6] Turn CI_ONLY_WHEN_CHANNEL from an environment variable to a job attribute --- src/ci/github-actions/calculate-job-matrix.py | 2 +- src/ci/github-actions/jobs.yml | 17 ++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py index 84c4ee40668..124c22bd979 100755 --- a/src/ci/github-actions/calculate-job-matrix.py +++ b/src/ci/github-actions/calculate-job-matrix.py @@ -95,7 +95,7 @@ def skip_jobs(jobs: List[Dict[str, Any]], channel: str) -> List[Job]: """ Skip CI jobs that are not supposed to be executed on the given `channel`. """ - return [j for j in jobs if j.get("CI_ONLY_WHEN_CHANNEL", channel) == channel] + return [j for j in jobs if j.get("only_on_channel", channel) == channel] def get_github_ctx() -> GitHubCtx: diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index 0f5dfaa5785..df6c5d6079d 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -200,24 +200,23 @@ auto: # channel name on the output), and this builder prevents landing # changes that would result in broken builds after a promotion. - image: x86_64-gnu-stable + # Only run this job on the nightly channel. Running this on beta + # could cause failures when `dev: 1` in `stage0.txt`, and running + # this on stable is useless. + only_on_channel: nightly env: IMAGE: x86_64-gnu RUST_CI_OVERRIDE_RELEASE_CHANNEL: stable - # Only run this job on the nightly channel. Running this on beta - # could cause failures when `dev: 1` in `stage0.txt`, and running - # this on stable is useless. - CI_ONLY_WHEN_CHANNEL: nightly <<: *job-linux-4c - image: x86_64-gnu-aux <<: *job-linux-4c - image: x86_64-gnu-integration - env: - # Only run this job on the nightly channel. Fuchsia requires - # nightly features to compile, and this job would fail if - # executed on beta and stable. - CI_ONLY_WHEN_CHANNEL: nightly + # Only run this job on the nightly channel. Fuchsia requires + # nightly features to compile, and this job would fail if + # executed on beta and stable. + only_on_channel: nightly <<: *job-linux-8c - image: x86_64-gnu-debug