mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Perform PR and try builds dynamically

Browse Source
This commit is contained in:
Jakub Beránek 2024-04-20 09:07:11 +02:00
parent 9fec43ddf1
commit 7a90679e28
No known key found for this signature in database
GPG Key ID: 909CD0D26483516B
4 changed files with 102 additions and 181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

155
.github/workflows/ci.yml vendored
View File

@ -45,23 +45,23 @@ jobs:
- name: Checkout the source code - name: Checkout the source code
uses: actions/checkout@v4 uses: actions/checkout@v4
- name: Calculate the CI job matrix - name: Calculate the CI job matrix
env:
GITHUB_CTX: "${{ toJSON(github) }}"
run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
id: jobs id: jobs
pr: job:
name: "PR - ${{ matrix.name }}" name: "${{ matrix.name }}"
needs: needs:
- calculate_matrix - calculate_matrix
env: env:
PR_CI_JOB: 1 CI_JOB_NAME: "${{ matrix.image }}"
CI_JOB_NAME: "${{ matrix.name }}"
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}" HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}" DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
SCCACHE_BUCKET: rust-lang-ci-sccache2 SCCACHE_BUCKET: rust-lang-ci-sccache2
TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate" TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
CACHE_DOMAIN: ci-caches.rust-lang.org CACHE_DOMAIN: ci-caches.rust-lang.org
if: "github.event_name == 'pull_request'" continue-on-error: "${{ matrix.continue_on_error || false }}"
continue-on-error: "${{ matrix.name == 'mingw-check-tidy' }}"
strategy: strategy:
matrix: matrix:
include: "${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}" include: "${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}"
@ -589,145 +589,6 @@ jobs:
AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}"
if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')" if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
try:
name: "try - ${{ matrix.name }}"
env:
DIST_TRY_BUILD: 1
CI_JOB_NAME: "${{ matrix.name }}"
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
SCCACHE_BUCKET: rust-lang-ci-sccache2
DEPLOY_BUCKET: rust-lang-ci2
TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
TOOLSTATE_ISSUES_API_URL: "https://api.github.com/repos/rust-lang/rust/issues"
TOOLSTATE_PUBLISH: 1
CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
AWS_REGION: us-west-1
CACHE_DOMAIN: ci-caches.rust-lang.org
if: "github.event_name == 'push' && (((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust') || ((github.ref == 'refs/heads/automation/bors/try') && github.repository == 'rust-lang/rust'))"
strategy:
matrix:
include:
- name: dist-x86_64-linux
env:
CODEGEN_BACKENDS: "llvm,cranelift"
os: ubuntu-20.04-16core-64gb
defaults:
run:
shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}"
timeout-minutes: 600
runs-on: "${{ matrix.os }}"
steps:
- if: "contains(matrix.os, 'windows')"
uses: msys2/setup-msys2@v2.22.0
with:
msystem: "${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}"
update: false
release: true
path-type: inherit
install: "make dos2unix diffutils\n"
- name: disable git crlf conversion
run: git config --global core.autocrlf false
- name: checkout the source code
uses: actions/checkout@v4
with:
fetch-depth: 2
- name: configure the PR in which the error message will be posted
run: "echo \"[CI_PR_NUMBER=$num]\""
env:
num: "${{ github.event.number }}"
if: "success() && !env.SKIP_JOB && github.event_name == 'pull_request'"
- name: add extra environment variables
run: src/ci/scripts/setup-environment.sh
env:
EXTRA_VARIABLES: "${{ toJson(matrix.env) }}"
if: success() && !env.SKIP_JOB
- name: decide whether to skip this job
run: src/ci/scripts/should-skip-this.sh
if: success() && !env.SKIP_JOB
- name: ensure the channel matches the target branch
run: src/ci/scripts/verify-channel.sh
if: success() && !env.SKIP_JOB
- name: collect CPU statistics
run: src/ci/scripts/collect-cpu-stats.sh
if: success() && !env.SKIP_JOB
- name: show the current environment
run: src/ci/scripts/dump-environment.sh
if: success() && !env.SKIP_JOB
- name: install awscli
run: src/ci/scripts/install-awscli.sh
if: success() && !env.SKIP_JOB
- name: install sccache
run: src/ci/scripts/install-sccache.sh
if: success() && !env.SKIP_JOB
- name: select Xcode
run: src/ci/scripts/select-xcode.sh
if: success() && !env.SKIP_JOB
- name: install clang
run: src/ci/scripts/install-clang.sh
if: success() && !env.SKIP_JOB
- name: install tidy
run: src/ci/scripts/install-tidy.sh
if: success() && !env.SKIP_JOB
- name: install WIX
run: src/ci/scripts/install-wix.sh
if: success() && !env.SKIP_JOB
- name: disable git crlf conversion
run: src/ci/scripts/disable-git-crlf-conversion.sh
if: success() && !env.SKIP_JOB
- name: checkout submodules
run: src/ci/scripts/checkout-submodules.sh
if: success() && !env.SKIP_JOB
- name: install MSYS2
run: src/ci/scripts/install-msys2.sh
if: success() && !env.SKIP_JOB
- name: install MinGW
run: src/ci/scripts/install-mingw.sh
if: success() && !env.SKIP_JOB
- name: install ninja
run: src/ci/scripts/install-ninja.sh
if: success() && !env.SKIP_JOB
- name: enable ipv6 on Docker
run: src/ci/scripts/enable-docker-ipv6.sh
if: success() && !env.SKIP_JOB
- name: disable git crlf conversion
run: src/ci/scripts/disable-git-crlf-conversion.sh
if: success() && !env.SKIP_JOB
- name: ensure line endings are correct
run: src/ci/scripts/verify-line-endings.sh
if: success() && !env.SKIP_JOB
- name: ensure backported commits are in upstream branches
run: src/ci/scripts/verify-backported-commits.sh
if: success() && !env.SKIP_JOB
- name: ensure the stable version number is correct
run: src/ci/scripts/verify-stable-version-number.sh
if: success() && !env.SKIP_JOB
- name: run the build
run: src/ci/scripts/run-build-from-ci.sh 2>&1
env:
AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
if: success() && !env.SKIP_JOB
- name: create github artifacts
run: src/ci/scripts/create-doc-artifacts.sh
if: success() && !env.SKIP_JOB
- name: upload artifacts to github
uses: actions/upload-artifact@v4
with:
name: "${{ env.DOC_ARTIFACT_NAME }}"
path: obj/artifacts/doc
if-no-files-found: ignore
retention-days: 5
if: success() && !env.SKIP_JOB
- name: upload artifacts to S3
run: src/ci/scripts/upload-artifacts.sh
env:
AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}"
if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
master: master:
name: master name: master
runs-on: ubuntu-latest runs-on: ubuntu-latest
@ -755,7 +616,7 @@ jobs:
if: success() && !env.SKIP_JOB if: success() && !env.SKIP_JOB
try-success: try-success:
needs: needs:
- try - job
if: "success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'" if: "success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
steps: steps:
- name: mark the job as a success - name: mark the job as a success
@ -765,7 +626,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
try-failure: try-failure:
needs: needs:
- try - job
if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'" if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
steps: steps:
- name: mark the job as a failure - name: mark the job as a failure

42
src/ci/github-actions/calculate-job-matrix.py
View File

@ -7,19 +7,53 @@ be executed on CI.
It reads job definitions from `src/ci/github-actions/jobs.yml` It reads job definitions from `src/ci/github-actions/jobs.yml`
and filters them based on the event that happened on CI. and filters them based on the event that happened on CI.
Currently, it only supports PR builds. Currently, it only supports PR and try builds.
""" """
import json import json
import os
import sys
from pathlib import Path from pathlib import Path
from typing import List, Dict
import yaml import yaml
JOBS_YAML_PATH = Path(__file__).absolute().parent / "jobs.yml" JOBS_YAML_PATH = Path(__file__).absolute().parent / "jobs.yml"
def name_jobs(jobs: List[Dict], prefix: str) -> List[Dict]:
for job in jobs:
job["name"] = f"{prefix} - {job['image']}"
return jobs
if __name__ == "__main__": if __name__ == "__main__":
github_ctx = json.loads(os.environ["GITHUB_CTX"])
with open(JOBS_YAML_PATH) as f: with open(JOBS_YAML_PATH) as f:
jobs = yaml.safe_load(f) data = yaml.safe_load(f)
job_output = jobs["pr"]
print(f"jobs={json.dumps(job_output)}") event_name = github_ctx["event_name"]
ref = github_ctx["ref"]
repository = github_ctx["repository"]
old_bors_try_build = (
ref in ("refs/heads/try", "refs/heads/try-perf") and
repository == "rust-lang-ci/rust"
)
new_bors_try_build = (
ref == "refs/heads/automation/bors/try" and
repository == "rust-lang/rust"
)
try_build = old_bors_try_build or new_bors_try_build
jobs = []
# Pull request CI jobs. Their name is 'PR - <image>'
if event_name == "pull_request":
jobs = name_jobs(data["pr"], "PR")
# Try builds
elif event_name == "push" and try_build:
jobs = name_jobs(data["try"], "try")
print(f"Output:\n{json.dumps(jobs, indent=4)}", file=sys.stderr)
print(f"jobs={json.dumps(jobs)}")

41
src/ci/github-actions/ci.yml
View File

@ -351,17 +351,25 @@ jobs:
- name: Checkout the source code - name: Checkout the source code
uses: actions/checkout@v4 uses: actions/checkout@v4
- name: Calculate the CI job matrix - name: Calculate the CI job matrix
env:
GITHUB_CTX: ${{ toJSON(github) }}
run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
id: jobs id: jobs
pr: # This is either a PR or a try job, decided by calculate-job-matrix.py
job:
<<: *base-ci-job <<: *base-ci-job
name: PR - ${{ matrix.name }} name: ${{ matrix.name }}
needs: [ calculate_matrix ] needs: [ calculate_matrix ]
env: env:
<<: [*shared-ci-variables, *public-variables] CI_JOB_NAME: ${{ matrix.image }}
PR_CI_JOB: 1 CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
if: github.event_name == 'pull_request' # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
continue-on-error: ${{ matrix.name == 'mingw-check-tidy' }} HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SCCACHE_BUCKET: rust-lang-ci-sccache2
TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
CACHE_DOMAIN: ci-caches.rust-lang.org
continue-on-error: ${{ matrix.continue_on_error || false }}
strategy: strategy:
matrix: matrix:
# Check the `calculate_matrix` job to see how is the matrix defined. # Check the `calculate_matrix` job to see how is the matrix defined.
@ -751,23 +759,6 @@ jobs:
SCRIPT: python x.py dist bootstrap --include-default-paths SCRIPT: python x.py dist bootstrap --include-default-paths
<<: *job-windows-8c <<: *job-windows-8c
try:
<<: *base-ci-job
name: try - ${{ matrix.name }}
env:
DIST_TRY_BUILD: 1
<<: [*shared-ci-variables, *prod-variables]
if: github.event_name == 'push' && (((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust') || ((github.ref == 'refs/heads/automation/bors/try') && github.repository == 'rust-lang/rust'))
strategy:
matrix:
include:
- &dist-x86_64-linux
name: dist-x86_64-linux
env:
CODEGEN_BACKENDS: llvm,cranelift
<<: *job-linux-16c
master: master:
name: master name: master
runs-on: ubuntu-latest runs-on: ubuntu-latest
@ -791,11 +782,11 @@ jobs:
# build completed, as there is no practical way to detect when a workflow is # build completed, as there is no practical way to detect when a workflow is
# successful listening to webhooks only. # successful listening to webhooks only.
try-success: try-success:
needs: [try] needs: [ job ]
if: "success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'" if: "success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
<<: *base-success-job <<: *base-success-job
try-failure: try-failure:
needs: [try] needs: [ job ]
if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'" if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
<<: *base-failure-job <<: *base-failure-job
auto-success: auto-success:

45
src/ci/github-actions/jobs.yml
View File

@ -2,7 +2,7 @@
# dynamically in CI from ci.yml. # dynamically in CI from ci.yml.
# You *do not* need to re-run `src/tools/expand-yaml-anchors` when you # You *do not* need to re-run `src/tools/expand-yaml-anchors` when you
# modify this file. # modify this file.
shared_defs: runners:
- &base-job - &base-job
env: { } env: { }
@ -37,14 +37,49 @@ shared_defs:
- &job-aarch64-linux - &job-aarch64-linux
os: [ self-hosted, ARM64, linux ] os: [ self-hosted, ARM64, linux ]
envs:
- &prod-variables
DEPLOY_BUCKET: rust-lang-ci2
TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues
TOOLSTATE_PUBLISH: 1
# AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
# AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
# rotate them in a single branch while keeping the old key in another
# branch, which wouldn't be possible if the key was named with the kind
# (caches, artifacts...).
CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
AWS_REGION: us-west-1
- &pr-env
PR_CI_JOB: 1
# Pull request jobs
pr: pr:
- name: mingw-check - image: mingw-check
env:
<<: *pr-env
<<: *job-linux-4c <<: *job-linux-4c
- name: mingw-check-tidy - image: mingw-check-tidy
continue_on_error: true
env:
<<: *pr-env
<<: *job-linux-4c <<: *job-linux-4c
- name: x86_64-gnu-llvm-17 - image: x86_64-gnu-llvm-17
env: env:
ENABLE_GCC_CODEGEN: "1" ENABLE_GCC_CODEGEN: "1"
<<: *pr-env
<<: *job-linux-16c <<: *job-linux-16c
- name: x86_64-gnu-tools - image: x86_64-gnu-tools
env:
<<: *pr-env
<<: *job-linux-16c
# Try build jobs
try:
- image: dist-x86_64-linux
env:
DIST_TRY_BUILD: 1
CODEGEN_BACKENDS: llvm,cranelift
<<: *prod-variables
<<: *job-linux-16c <<: *job-linux-16c