Rollup merge of #124327 - Kobzol:ci-skip-jobs, r=pietroalbini

CI: implement job skipping in Python matrix calculation

This removes the `step` YAML anchor and the corresponding bash script.

Best reviewed commit-by-commit.

r? ```@pietroalbini```

.github/workflows/ci.yml

@@ -88,82 +88,57 @@ jobs:
         run: "echo \"[CI_PR_NUMBER=$num]\""
         env:
           num: "${{ github.event.number }}"
-        if: "success() && !env.SKIP_JOB && github.event_name == 'pull_request'"
+        if: "success() && github.event_name == 'pull_request'"
       - name: add extra environment variables
         run: src/ci/scripts/setup-environment.sh
         env:
           EXTRA_VARIABLES: "${{ toJson(matrix.env) }}"
-        if: success() && !env.SKIP_JOB
-      - name: decide whether to skip this job
-        run: src/ci/scripts/should-skip-this.sh
-        if: success() && !env.SKIP_JOB
       - name: ensure the channel matches the target branch
         run: src/ci/scripts/verify-channel.sh
-        if: success() && !env.SKIP_JOB
       - name: collect CPU statistics
         run: src/ci/scripts/collect-cpu-stats.sh
-        if: success() && !env.SKIP_JOB
       - name: show the current environment
         run: src/ci/scripts/dump-environment.sh
-        if: success() && !env.SKIP_JOB
       - name: install awscli
         run: src/ci/scripts/install-awscli.sh
-        if: success() && !env.SKIP_JOB
       - name: install sccache
         run: src/ci/scripts/install-sccache.sh
-        if: success() && !env.SKIP_JOB
       - name: select Xcode
         run: src/ci/scripts/select-xcode.sh
-        if: success() && !env.SKIP_JOB
       - name: install clang
         run: src/ci/scripts/install-clang.sh
-        if: success() && !env.SKIP_JOB
       - name: install tidy
         run: src/ci/scripts/install-tidy.sh
-        if: success() && !env.SKIP_JOB
       - name: install WIX
         run: src/ci/scripts/install-wix.sh
-        if: success() && !env.SKIP_JOB
       - name: disable git crlf conversion
         run: src/ci/scripts/disable-git-crlf-conversion.sh
-        if: success() && !env.SKIP_JOB
       - name: checkout submodules
         run: src/ci/scripts/checkout-submodules.sh
-        if: success() && !env.SKIP_JOB
       - name: install MSYS2
         run: src/ci/scripts/install-msys2.sh
-        if: success() && !env.SKIP_JOB
       - name: install MinGW
         run: src/ci/scripts/install-mingw.sh
-        if: success() && !env.SKIP_JOB
       - name: install ninja
         run: src/ci/scripts/install-ninja.sh
-        if: success() && !env.SKIP_JOB
       - name: enable ipv6 on Docker
         run: src/ci/scripts/enable-docker-ipv6.sh
-        if: success() && !env.SKIP_JOB
       - name: disable git crlf conversion
         run: src/ci/scripts/disable-git-crlf-conversion.sh
-        if: success() && !env.SKIP_JOB
       - name: ensure line endings are correct
         run: src/ci/scripts/verify-line-endings.sh
-        if: success() && !env.SKIP_JOB
       - name: ensure backported commits are in upstream branches
         run: src/ci/scripts/verify-backported-commits.sh
-        if: success() && !env.SKIP_JOB
       - name: ensure the stable version number is correct
         run: src/ci/scripts/verify-stable-version-number.sh
-        if: success() && !env.SKIP_JOB
       - name: run the build
         run: src/ci/scripts/run-build-from-ci.sh 2>&1
         env:
           AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}"
           AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
           TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
-        if: success() && !env.SKIP_JOB
       - name: create github artifacts
         run: src/ci/scripts/create-doc-artifacts.sh
-        if: success() && !env.SKIP_JOB
       - name: upload artifacts to github
         uses: actions/upload-artifact@v4
         with:
@@ -171,13 +146,12 @@ jobs:
           path: obj/artifacts/doc
           if-no-files-found: ignore
           retention-days: 5
-        if: success() && !env.SKIP_JOB
       - name: upload artifacts to S3
         run: src/ci/scripts/upload-artifacts.sh
         env:
           AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}"
           AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}"
-        if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
+        if: "success() && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
   master:
     name: master
     runs-on: ubuntu-latest
@@ -202,7 +176,6 @@ jobs:
         shell: bash
         env:
           TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
-        if: success() && !env.SKIP_JOB
   try-success:
     needs:
       - job

src/ci/github-actions/calculate-job-matrix.py

@@ -17,10 +17,13 @@ from typing import List, Dict, Any, Optional
 
 import yaml
 
+CI_DIR = Path(__file__).absolute().parent.parent
 JOBS_YAML_PATH = Path(__file__).absolute().parent / "jobs.yml"
 
+Job = Dict[str, Any]
 
-def name_jobs(jobs: List[Dict], prefix: str) -> List[Dict]:
+
+def name_jobs(jobs: List[Dict], prefix: str) -> List[Job]:
     """
     Add a `name` attribute to each job, based on its image and the given `prefix`.
     """
@@ -29,7 +32,7 @@ def name_jobs(jobs: List[Dict], prefix: str) -> List[Dict]:
     return jobs
 
 
-def add_base_env(jobs: List[Dict], environment: Dict[str, str]) -> List[Dict]:
+def add_base_env(jobs: List[Job], environment: Dict[str, str]) -> List[Job]:
     """
     Prepends `environment` to the `env` attribute of each job.
     The `env` of each job has higher precedence than `environment`.
@@ -77,7 +80,7 @@ def find_job_type(ctx: GitHubCtx) -> Optional[JobType]:
     return None
 
 
-def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Dict[str, Any]]:
+def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Job]:
     if job_type == JobType.PR:
         return add_base_env(name_jobs(job_data["pr"], "PR"), job_data["envs"]["pr"])
     elif job_type == JobType.Try:
@@ -88,6 +91,13 @@ def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Dict[str
     return []
 
 
+def skip_jobs(jobs: List[Dict[str, Any]], channel: str) -> List[Job]:
+    """
+    Skip CI jobs that are not supposed to be executed on the given `channel`.
+    """
+    return [j for j in jobs if j.get("only_on_channel", channel) == channel]
+
+
 def get_github_ctx() -> GitHubCtx:
     return GitHubCtx(
         event_name=os.environ["GITHUB_EVENT_NAME"],
@@ -107,9 +117,13 @@ if __name__ == "__main__":
     job_type = find_job_type(github_ctx)
     logging.info(f"Job type: {job_type}")
 
+    with open(CI_DIR / "channel") as f:
+        channel = f.read().strip()
+
     jobs = []
     if job_type is not None:
         jobs = calculate_jobs(job_type, data)
+    jobs = skip_jobs(jobs, channel)
 
     logging.info(f"Output:\n{yaml.dump(jobs, indent=4)}")
     print(f"jobs={json.dumps(jobs)}")

src/ci/github-actions/ci.yml

@@ -107,9 +107,6 @@ x--expand-yaml-anchors--remove:
   - &job-aarch64-linux
     os: [self-hosted, ARM64, linux]
 
-  - &step
-    if: success() && !env.SKIP_JOB
-
   - &base-ci-job
     defaults:
       run:
@@ -151,7 +148,7 @@ x--expand-yaml-anchors--remove:
         run: echo "[CI_PR_NUMBER=$num]"
         env:
           num: ${{ github.event.number }}
-        if: success() && !env.SKIP_JOB && github.event_name == 'pull_request'
+        if: success() && github.event_name == 'pull_request'
 
       - name: add extra environment variables
         run: src/ci/scripts/setup-environment.sh
@@ -161,71 +158,51 @@ x--expand-yaml-anchors--remove:
           # are passed to the `setup-environment.sh` script encoded in JSON,
           # which then uses log commands to actually set them.
           EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
-        <<: *step
-
-      - name: decide whether to skip this job
-        run: src/ci/scripts/should-skip-this.sh
-        <<: *step
 
       - name: ensure the channel matches the target branch
         run: src/ci/scripts/verify-channel.sh
-        <<: *step
 
       - name: collect CPU statistics
         run: src/ci/scripts/collect-cpu-stats.sh
-        <<: *step
 
       - name: show the current environment
         run: src/ci/scripts/dump-environment.sh
-        <<: *step
 
       - name: install awscli
         run: src/ci/scripts/install-awscli.sh
-        <<: *step
 
       - name: install sccache
         run: src/ci/scripts/install-sccache.sh
-        <<: *step
 
       - name: select Xcode
         run: src/ci/scripts/select-xcode.sh
-        <<: *step
 
       - name: install clang
         run: src/ci/scripts/install-clang.sh
-        <<: *step
 
       - name: install tidy
         run: src/ci/scripts/install-tidy.sh
-        <<: *step
 
       - name: install WIX
         run: src/ci/scripts/install-wix.sh
-        <<: *step
 
       - name: disable git crlf conversion
         run: src/ci/scripts/disable-git-crlf-conversion.sh
-        <<: *step
 
       - name: checkout submodules
         run: src/ci/scripts/checkout-submodules.sh
-        <<: *step
 
       - name: install MSYS2
         run: src/ci/scripts/install-msys2.sh
-        <<: *step
 
       - name: install MinGW
         run: src/ci/scripts/install-mingw.sh
-        <<: *step
 
       - name: install ninja
         run: src/ci/scripts/install-ninja.sh
-        <<: *step
 
       - name: enable ipv6 on Docker
         run: src/ci/scripts/enable-docker-ipv6.sh
-        <<: *step
 
       # Disable automatic line ending conversion (again). On Windows, when we're
       # installing dependencies, something switches the git configuration directory or
@@ -234,19 +211,15 @@ x--expand-yaml-anchors--remove:
       # appropriate line endings.
       - name: disable git crlf conversion
         run: src/ci/scripts/disable-git-crlf-conversion.sh
-        <<: *step
 
       - name: ensure line endings are correct
         run: src/ci/scripts/verify-line-endings.sh
-        <<: *step
 
       - name: ensure backported commits are in upstream branches
         run: src/ci/scripts/verify-backported-commits.sh
-        <<: *step
 
       - name: ensure the stable version number is correct
         run: src/ci/scripts/verify-stable-version-number.sh
-        <<: *step
 
       - name: run the build
         # Redirect stderr to stdout to avoid reordering the two streams in the GHA logs.
@@ -255,11 +228,9 @@ x--expand-yaml-anchors--remove:
           AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
           TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
-        <<: *step
 
       - name: create github artifacts
         run: src/ci/scripts/create-doc-artifacts.sh
-        <<: *step
 
       - name: upload artifacts to github
         uses: actions/upload-artifact@v4
@@ -269,7 +240,6 @@ x--expand-yaml-anchors--remove:
           path: obj/artifacts/doc
           if-no-files-found: ignore
           retention-days: 5
-        <<: *step
 
       - name: upload artifacts to S3
         run: src/ci/scripts/upload-artifacts.sh
@@ -281,8 +251,7 @@ x--expand-yaml-anchors--remove:
         # adding the condition is helpful as this way CI will not silently skip
         # deploying artifacts from a dist builder if the variables are misconfigured,
         # erroring about invalid credentials instead.
-        if: success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')
+        if: success() && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')
-        <<: *step
 
   # These snippets are used by the try-success, try-failure, auto-success and auto-failure jobs.
   # Check out their documentation for more information on why they're needed.
@@ -399,7 +368,6 @@ jobs:
         shell: bash
         env:
           TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
-        <<: *step
 
   # These jobs don't actually test anything, but they're used to tell bors the
   # build completed, as there is no practical way to detect when a workflow is

src/ci/github-actions/jobs.yml

@@ -79,7 +79,7 @@ pr:
     <<: *job-linux-16c
 
 # Jobs that run when you perform a try build (@bors try)
-# These jobs automatically inherit envs.production, to avoid repeating
+# These jobs automatically inherit envs.try, to avoid repeating
 # it in each job definition.
 try:
   - image: dist-x86_64-linux
@@ -88,7 +88,7 @@ try:
     <<: *job-linux-16c
 
 # Main CI jobs that have to be green to merge a commit into master
-# These jobs automatically inherit envs.production, to avoid repeating
+# These jobs automatically inherit envs.auto, to avoid repeating
 # it in each job definition.
 auto:
   #############################
@@ -200,24 +200,23 @@ auto:
   # channel name on the output), and this builder prevents landing
   # changes that would result in broken builds after a promotion.
   - image: x86_64-gnu-stable
+    # Only run this job on the nightly channel. Running this on beta
+    # could cause failures when `dev: 1` in `stage0.txt`, and running
+    # this on stable is useless.
+    only_on_channel: nightly
     env:
       IMAGE: x86_64-gnu
       RUST_CI_OVERRIDE_RELEASE_CHANNEL: stable
-      # Only run this job on the nightly channel. Running this on beta
-      # could cause failures when `dev: 1` in `stage0.txt`, and running
-      # this on stable is useless.
-      CI_ONLY_WHEN_CHANNEL: nightly
     <<: *job-linux-4c
 
   - image: x86_64-gnu-aux
     <<: *job-linux-4c
 
   - image: x86_64-gnu-integration
-    env:
+    # Only run this job on the nightly channel. Fuchsia requires
-      # Only run this job on the nightly channel. Fuchsia requires
+    # nightly features to compile, and this job would fail if
-      # nightly features to compile, and this job would fail if
+    # executed on beta and stable.
-      # executed on beta and stable.
+    only_on_channel: nightly
-      CI_ONLY_WHEN_CHANNEL: nightly
     <<: *job-linux-8c
 
   - image: x86_64-gnu-debug

src/ci/scripts/should-skip-this.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-# Set the SKIP_JOB environment variable if this job is not supposed to run on the current builder.
-
-set -euo pipefail
-IFS=$'\n\t'
-
-source "$(cd "$(dirname "$0")" && pwd)/../shared.sh"
-
-if [[ -n "${CI_ONLY_WHEN_CHANNEL-}" ]]; then
-    if [[ "${CI_ONLY_WHEN_CHANNEL}" = "$(cat src/ci/channel)" ]]; then
-        echo "The channel is the expected one"
-    else
-        echo "Not executing this job as the channel is not the expected one"
-        ciCommandSetEnv SKIP_JOB 1
-        exit 0
-    fi
-fi
-
-
-echo "Executing the job since there is no skip rule preventing the execution"
-exit 0