mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rollup merge of #124327 - Kobzol:ci-skip-jobs, r=pietroalbini

Browse Source 
CI: implement job skipping in Python matrix calculation

This removes the `step` YAML anchor and the corresponding bash script.

Best reviewed commit-by-commit.

r? ```@pietroalbini```
This commit is contained in:
Matthias Krüger 2024-04-25 06:31:04 +02:00 committed by GitHub
commit 036bf570ad
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 31 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
.github/workflows/ci.yml vendored
View File

@ -88,82 +88,57 @@ jobs:
run: "echo \"[CI_PR_NUMBER=$num]\""
env:
num: "${{ github.event.number }}"
if: "success() && !env.SKIP_JOB && github.event_name == 'pull_request'"
if: "success() && github.event_name == 'pull_request'"
- name: add extra environment variables
run: src/ci/scripts/setup-environment.sh
env:
EXTRA_VARIABLES: "${{ toJson(matrix.env) }}"
if: success() && !env.SKIP_JOB
- name: decide whether to skip this job
run: src/ci/scripts/should-skip-this.sh
if: success() && !env.SKIP_JOB
- name: ensure the channel matches the target branch
run: src/ci/scripts/verify-channel.sh
if: success() && !env.SKIP_JOB
- name: collect CPU statistics
run: src/ci/scripts/collect-cpu-stats.sh
if: success() && !env.SKIP_JOB
- name: show the current environment
run: src/ci/scripts/dump-environment.sh
if: success() && !env.SKIP_JOB
- name: install awscli
run: src/ci/scripts/install-awscli.sh
if: success() && !env.SKIP_JOB
- name: install sccache
run: src/ci/scripts/install-sccache.sh
if: success() && !env.SKIP_JOB
- name: select Xcode
run: src/ci/scripts/select-xcode.sh
if: success() && !env.SKIP_JOB
- name: install clang
run: src/ci/scripts/install-clang.sh
if: success() && !env.SKIP_JOB
- name: install tidy
run: src/ci/scripts/install-tidy.sh
if: success() && !env.SKIP_JOB
- name: install WIX
run: src/ci/scripts/install-wix.sh
if: success() && !env.SKIP_JOB
- name: disable git crlf conversion
run: src/ci/scripts/disable-git-crlf-conversion.sh
if: success() && !env.SKIP_JOB
- name: checkout submodules
run: src/ci/scripts/checkout-submodules.sh
if: success() && !env.SKIP_JOB
- name: install MSYS2
run: src/ci/scripts/install-msys2.sh
if: success() && !env.SKIP_JOB
- name: install MinGW
run: src/ci/scripts/install-mingw.sh
if: success() && !env.SKIP_JOB
- name: install ninja
run: src/ci/scripts/install-ninja.sh
if: success() && !env.SKIP_JOB
- name: enable ipv6 on Docker
run: src/ci/scripts/enable-docker-ipv6.sh
if: success() && !env.SKIP_JOB
- name: disable git crlf conversion
run: src/ci/scripts/disable-git-crlf-conversion.sh
if: success() && !env.SKIP_JOB
- name: ensure line endings are correct
run: src/ci/scripts/verify-line-endings.sh
if: success() && !env.SKIP_JOB
- name: ensure backported commits are in upstream branches
run: src/ci/scripts/verify-backported-commits.sh
if: success() && !env.SKIP_JOB
- name: ensure the stable version number is correct
run: src/ci/scripts/verify-stable-version-number.sh
if: success() && !env.SKIP_JOB
- name: run the build
run: src/ci/scripts/run-build-from-ci.sh 2>&1
env:
AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
if: success() && !env.SKIP_JOB
- name: create github artifacts
run: src/ci/scripts/create-doc-artifacts.sh
if: success() && !env.SKIP_JOB
- name: upload artifacts to github
uses: actions/upload-artifact@v4
with:
@ -171,13 +146,12 @@ jobs:
path: obj/artifacts/doc
if-no-files-found: ignore
retention-days: 5
if: success() && !env.SKIP_JOB
- name: upload artifacts to S3
run: src/ci/scripts/upload-artifacts.sh
env:
AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}"
if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
if: "success() && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
master:
name: master
runs-on: ubuntu-latest
@ -202,7 +176,6 @@ jobs:
shell: bash
env:
TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
if: success() && !env.SKIP_JOB
try-success:
needs:
- job

20
src/ci/github-actions/calculate-job-matrix.py
View File

@ -17,10 +17,13 @@ from typing import List, Dict, Any, Optional
import yaml
CI_DIR = Path(__file__).absolute().parent.parent
JOBS_YAML_PATH = Path(__file__).absolute().parent / "jobs.yml"
Job = Dict[str, Any]
def name_jobs(jobs: List[Dict], prefix: str) -> List[Dict]:
def name_jobs(jobs: List[Dict], prefix: str) -> List[Job]:
"""
Add a `name` attribute to each job, based on its image and the given `prefix`.
"""
@ -29,7 +32,7 @@ def name_jobs(jobs: List[Dict], prefix: str) -> List[Dict]:
return jobs
def add_base_env(jobs: List[Dict], environment: Dict[str, str]) -> List[Dict]:
def add_base_env(jobs: List[Job], environment: Dict[str, str]) -> List[Job]:
"""
Prepends `environment` to the `env` attribute of each job.
The `env` of each job has higher precedence than `environment`.
@ -77,7 +80,7 @@ def find_job_type(ctx: GitHubCtx) -> Optional[JobType]:
return None
def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Dict[str, Any]]:
def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Job]:
if job_type == JobType.PR:
return add_base_env(name_jobs(job_data["pr"], "PR"), job_data["envs"]["pr"])
elif job_type == JobType.Try:
@ -88,6 +91,13 @@ def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Dict[str
return []
def skip_jobs(jobs: List[Dict[str, Any]], channel: str) -> List[Job]:
"""
Skip CI jobs that are not supposed to be executed on the given `channel`.
"""
return [j for j in jobs if j.get("only_on_channel", channel) == channel]
def get_github_ctx() -> GitHubCtx:
return GitHubCtx(
event_name=os.environ["GITHUB_EVENT_NAME"],
@ -107,9 +117,13 @@ if __name__ == "__main__":
job_type = find_job_type(github_ctx)
logging.info(f"Job type: {job_type}")
with open(CI_DIR / "channel") as f:
channel = f.read().strip()
jobs = []
if job_type is not None:
jobs = calculate_jobs(job_type, data)
jobs = skip_jobs(jobs, channel)
logging.info(f"Output:\n{yaml.dump(jobs, indent=4)}")
print(f"jobs={json.dumps(jobs)}")

36
src/ci/github-actions/ci.yml
View File

@ -107,9 +107,6 @@ x--expand-yaml-anchors--remove:
- &job-aarch64-linux
os: [self-hosted, ARM64, linux]
- &step
if: success() && !env.SKIP_JOB
- &base-ci-job
defaults:
run:
@ -151,7 +148,7 @@ x--expand-yaml-anchors--remove:
run: echo "[CI_PR_NUMBER=$num]"
env:
num: ${{ github.event.number }}
if: success() && !env.SKIP_JOB && github.event_name == 'pull_request'
if: success() && github.event_name == 'pull_request'
- name: add extra environment variables
run: src/ci/scripts/setup-environment.sh
@ -161,71 +158,51 @@ x--expand-yaml-anchors--remove:
# are passed to the `setup-environment.sh` script encoded in JSON,
# which then uses log commands to actually set them.
EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
<<: *step
- name: decide whether to skip this job
run: src/ci/scripts/should-skip-this.sh
<<: *step
- name: ensure the channel matches the target branch
run: src/ci/scripts/verify-channel.sh
<<: *step
- name: collect CPU statistics
run: src/ci/scripts/collect-cpu-stats.sh
<<: *step
- name: show the current environment
run: src/ci/scripts/dump-environment.sh
<<: *step
- name: install awscli
run: src/ci/scripts/install-awscli.sh
<<: *step
- name: install sccache
run: src/ci/scripts/install-sccache.sh
<<: *step
- name: select Xcode
run: src/ci/scripts/select-xcode.sh
<<: *step
- name: install clang
run: src/ci/scripts/install-clang.sh
<<: *step
- name: install tidy
run: src/ci/scripts/install-tidy.sh
<<: *step
- name: install WIX
run: src/ci/scripts/install-wix.sh
<<: *step
- name: disable git crlf conversion
run: src/ci/scripts/disable-git-crlf-conversion.sh
<<: *step
- name: checkout submodules
run: src/ci/scripts/checkout-submodules.sh
<<: *step
- name: install MSYS2
run: src/ci/scripts/install-msys2.sh
<<: *step
- name: install MinGW
run: src/ci/scripts/install-mingw.sh
<<: *step
- name: install ninja
run: src/ci/scripts/install-ninja.sh
<<: *step
- name: enable ipv6 on Docker
run: src/ci/scripts/enable-docker-ipv6.sh
<<: *step
# Disable automatic line ending conversion (again). On Windows, when we're
# installing dependencies, something switches the git configuration directory or
@ -234,19 +211,15 @@ x--expand-yaml-anchors--remove:
# appropriate line endings.
- name: disable git crlf conversion
run: src/ci/scripts/disable-git-crlf-conversion.sh
<<: *step
- name: ensure line endings are correct
run: src/ci/scripts/verify-line-endings.sh
<<: *step
- name: ensure backported commits are in upstream branches
run: src/ci/scripts/verify-backported-commits.sh
<<: *step
- name: ensure the stable version number is correct
run: src/ci/scripts/verify-stable-version-number.sh
<<: *step
- name: run the build
# Redirect stderr to stdout to avoid reordering the two streams in the GHA logs.
@ -255,11 +228,9 @@ x--expand-yaml-anchors--remove:
AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
<<: *step
- name: create github artifacts
run: src/ci/scripts/create-doc-artifacts.sh
<<: *step
- name: upload artifacts to github
uses: actions/upload-artifact@v4
@ -269,7 +240,6 @@ x--expand-yaml-anchors--remove:
path: obj/artifacts/doc
if-no-files-found: ignore
retention-days: 5
<<: *step
- name: upload artifacts to S3
run: src/ci/scripts/upload-artifacts.sh
@ -281,8 +251,7 @@ x--expand-yaml-anchors--remove:
# adding the condition is helpful as this way CI will not silently skip
# deploying artifacts from a dist builder if the variables are misconfigured,
# erroring about invalid credentials instead.
if: success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')
<<: *step
if: success() && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')
# These snippets are used by the try-success, try-failure, auto-success and auto-failure jobs.
# Check out their documentation for more information on why they're needed.
@ -399,7 +368,6 @@ jobs:
shell: bash
env:
TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
<<: *step
# These jobs don't actually test anything, but they're used to tell bors the
# build completed, as there is no practical way to detect when a workflow is

15
src/ci/github-actions/jobs.yml
View File

@ -79,7 +79,7 @@ pr:
<<: *job-linux-16c
# Jobs that run when you perform a try build (@bors try)
# These jobs automatically inherit envs.production, to avoid repeating
# These jobs automatically inherit envs.try, to avoid repeating
# it in each job definition.
try:
- image: dist-x86_64-linux
@ -88,7 +88,7 @@ try:
<<: *job-linux-16c
# Main CI jobs that have to be green to merge a commit into master
# These jobs automatically inherit envs.production, to avoid repeating
# These jobs automatically inherit envs.auto, to avoid repeating
# it in each job definition.
auto:
#############################
@ -200,24 +200,23 @@ auto:
# channel name on the output), and this builder prevents landing
# changes that would result in broken builds after a promotion.
- image: x86_64-gnu-stable
env:
IMAGE: x86_64-gnu
RUST_CI_OVERRIDE_RELEASE_CHANNEL: stable
# Only run this job on the nightly channel. Running this on beta
# could cause failures when `dev: 1` in `stage0.txt`, and running
# this on stable is useless.
CI_ONLY_WHEN_CHANNEL: nightly
only_on_channel: nightly
env:
IMAGE: x86_64-gnu
RUST_CI_OVERRIDE_RELEASE_CHANNEL: stable
<<: *job-linux-4c
- image: x86_64-gnu-aux
<<: *job-linux-4c
- image: x86_64-gnu-integration
env:
# Only run this job on the nightly channel. Fuchsia requires
# nightly features to compile, and this job would fail if
# executed on beta and stable.
CI_ONLY_WHEN_CHANNEL: nightly
only_on_channel: nightly
<<: *job-linux-8c
- image: x86_64-gnu-debug

21
src/ci/scripts/should-skip-this.sh
View File

@ -1,21 +0,0 @@
#!/bin/bash
# Set the SKIP_JOB environment variable if this job is not supposed to run on the current builder.
set -euo pipefail
IFS=$'\n\t'
source "$(cd "$(dirname "$0")" && pwd)/../shared.sh"
if [[ -n "${CI_ONLY_WHEN_CHANNEL-}" ]]; then
if [[ "${CI_ONLY_WHEN_CHANNEL}" = "$(cat src/ci/channel)" ]]; then
echo "The channel is the expected one"
else
echo "Not executing this job as the channel is not the expected one"
ciCommandSetEnv SKIP_JOB 1
exit 0
fi
fi
echo "Executing the job since there is no skip rule preventing the execution"
exit 0