diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cfcbb8656b1..6fdc8d4d72d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,21 +1,14 @@ -############################################################# -# WARNING: automatically generated file, DO NOT CHANGE! # -############################################################# +# This file defines our primary CI workflow that runs on pull requests +# and also on pushes to special branches (auto, try). +# +# The actual definition of the executed jobs is calculated by a Python +# script located at src/ci/github-actions/calculate-job-matrix.py, which +# uses job definition data from src/ci/github-actions/jobs.yml. +# You should primarily modify the `jobs.yml` file if you want to modify +# what jobs are executed in CI. -# This file was automatically generated by the expand-yaml-anchors tool. The -# source file that generated this one is: -# -# src/ci/github-actions/ci.yml -# -# Once you make changes to that file you need to run: -# -# ./x.py run src/tools/expand-yaml-anchors/ -# -# The CI build will fail if the tool is not run after changes to this file. - ---- name: CI -"on": +on: push: branches: - auto @@ -25,23 +18,35 @@ name: CI pull_request: branches: - "**" + permissions: contents: read packages: write + defaults: run: + # On Linux, macOS, and Windows, use the system-provided bash as the default + # shell. (This should only make a difference on Windows, where the default + # shell is PowerShell.) shell: bash + concurrency: - group: "${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }}" + # For a given workflow, if we push to the same branch, cancel all previous builds on that branch. + # We add an exception for try builds (try branch) and unrolled rollup builds (try-perf), which + # are all triggered on the same branch, but which should be able to run concurrently. + group: ${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }} cancel-in-progress: true env: TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate" jobs: + # The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml. + # It calculates which jobs should be executed, based on the data of the ${{ github }} context. + # If you want to modify CI jobs, take a look at src/ci/github-actions/jobs.yml. calculate_matrix: name: Calculate job matrix runs-on: ubuntu-latest outputs: - jobs: "${{ steps.jobs.outputs.jobs }}" + jobs: ${{ steps.jobs.outputs.jobs }} steps: - name: Checkout the source code uses: actions/checkout@v4 @@ -49,128 +54,195 @@ jobs: run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT id: jobs job: - name: "${{ matrix.name }}" - needs: - - calculate_matrix + name: ${{ matrix.name }} + needs: [ calculate_matrix ] runs-on: "${{ matrix.os }}" defaults: run: - shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}" + shell: ${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }} timeout-minutes: 600 env: - CI_JOB_NAME: "${{ matrix.image }}" + CI_JOB_NAME: ${{ matrix.image }} CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse - HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}" - DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs. + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }} SCCACHE_BUCKET: rust-lang-ci-sccache2 CACHE_DOMAIN: ci-caches.rust-lang.org - continue-on-error: "${{ matrix.continue_on_error || false }}" + continue-on-error: ${{ matrix.continue_on_error || false }} strategy: matrix: - include: "${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}" - if: "fromJSON(needs.calculate_matrix.outputs.jobs)[0] != null" + # Check the `calculate_matrix` job to see how is the matrix defined. + include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }} + # GitHub Actions fails the workflow if an empty list of jobs is provided to + # the workflow, so we need to skip this job if nothing was produced by + # the Python script. + # + # Unfortunately checking whether a list is empty is not possible in a nice + # way due to GitHub Actions expressions limits. + # This hack is taken from https://github.com/ferrocene/ferrocene/blob/d43edc6b7697cf1719ec1c17c54904ab94825763/.github/workflows/release.yml#L75-L82 + if: fromJSON(needs.calculate_matrix.outputs.jobs)[0] != null steps: - - if: "contains(matrix.os, 'windows')" + - if: contains(matrix.os, 'windows') uses: msys2/setup-msys2@v2.22.0 with: - msystem: "${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}" + # i686 jobs use mingw32. x86_64 and cross-compile jobs use mingw64. + msystem: ${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }} + # don't try to download updates for already installed packages update: false + # don't try to use the msys that comes built-in to the github runner, + # so we can control what is installed (i.e. not python) release: true + # Inherit the full path from the Windows environment, with MSYS2's */bin/ + # dirs placed in front. This lets us run Windows-native Python etc. path-type: inherit - install: "make dos2unix diffutils\n" + install: > + make + dos2unix + diffutils + - name: disable git crlf conversion run: git config --global core.autocrlf false + - name: checkout the source code uses: actions/checkout@v4 with: fetch-depth: 2 + + # Rust Log Analyzer can't currently detect the PR number of a GitHub + # Actions build on its own, so a hint in the log message is needed to + # point it in the right direction. - name: configure the PR in which the error message will be posted - run: "echo \"[CI_PR_NUMBER=$num]\"" + run: echo "[CI_PR_NUMBER=$num]" env: - num: "${{ github.event.number }}" - if: "success() && github.event_name == 'pull_request'" + num: ${{ github.event.number }} + if: success() && github.event_name == 'pull_request' + - name: add extra environment variables run: src/ci/scripts/setup-environment.sh env: - EXTRA_VARIABLES: "${{ toJson(matrix.env) }}" + # Since it's not possible to merge `${{ matrix.env }}` with the other + # variables in `job..env`, the variables defined in the matrix + # are passed to the `setup-environment.sh` script encoded in JSON, + # which then uses log commands to actually set them. + EXTRA_VARIABLES: ${{ toJson(matrix.env) }} + - name: ensure the channel matches the target branch run: src/ci/scripts/verify-channel.sh + - name: collect CPU statistics run: src/ci/scripts/collect-cpu-stats.sh + - name: show the current environment run: src/ci/scripts/dump-environment.sh + - name: install awscli run: src/ci/scripts/install-awscli.sh + - name: install sccache run: src/ci/scripts/install-sccache.sh + - name: select Xcode run: src/ci/scripts/select-xcode.sh + - name: install clang run: src/ci/scripts/install-clang.sh + - name: install tidy run: src/ci/scripts/install-tidy.sh + - name: install WIX run: src/ci/scripts/install-wix.sh + - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh + - name: checkout submodules run: src/ci/scripts/checkout-submodules.sh + - name: install MSYS2 run: src/ci/scripts/install-msys2.sh + - name: install MinGW run: src/ci/scripts/install-mingw.sh + - name: install ninja run: src/ci/scripts/install-ninja.sh + - name: enable ipv6 on Docker run: src/ci/scripts/enable-docker-ipv6.sh + + # Disable automatic line ending conversion (again). On Windows, when we're + # installing dependencies, something switches the git configuration directory or + # re-enables autocrlf. We've not tracked down the exact cause -- and there may + # be multiple -- but this should ensure submodules are checked out with the + # appropriate line endings. - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh + - name: ensure line endings are correct run: src/ci/scripts/verify-line-endings.sh + - name: ensure backported commits are in upstream branches run: src/ci/scripts/verify-backported-commits.sh + - name: ensure the stable version number is correct run: src/ci/scripts/verify-stable-version-number.sh + - name: run the build + # Redirect stderr to stdout to avoid reordering the two streams in the GHA logs. run: src/ci/scripts/run-build-from-ci.sh 2>&1 env: - AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}" - AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" - TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" + AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }} + TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }} + - name: create github artifacts run: src/ci/scripts/create-doc-artifacts.sh + - name: upload artifacts to github uses: actions/upload-artifact@v4 with: - name: "${{ env.DOC_ARTIFACT_NAME }}" + # name is set in previous step + name: ${{ env.DOC_ARTIFACT_NAME }} path: obj/artifacts/doc if-no-files-found: ignore retention-days: 5 + - name: upload artifacts to S3 run: src/ci/scripts/upload-artifacts.sh env: - AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" - AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" - if: "success() && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')" + AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }} + # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy + # builders *should* have the AWS credentials available. Still, explicitly + # adding the condition is helpful as this way CI will not silently skip + # deploying artifacts from a dist builder if the variables are misconfigured, + # erroring about invalid credentials instead. + if: success() && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1') + + # This job isused to tell bors the final status of the build, as there is no practical way to detect + # when a workflow is successful listening to webhooks only in our current bors implementation (homu). outcome: name: bors build finished runs-on: ubuntu-latest - needs: - - job + needs: [ job ] + # !cancelled() executes the job regardless of whether the previous jobs passed or failed if: "!cancelled() && github.event_name == 'push'" steps: - name: checkout the source code uses: actions/checkout@v4 with: fetch-depth: 2 + # Calculate the exit status of the whole CI workflow. + # If all dependent jobs were successful, this exits with 0 (and the outcome job continues successfully). + # If a some dependent job has failed, this exits with 1. - name: calculate the correct exit status - id: status - run: "jq --exit-status 'all(.result == \"success\" or .result == \"skipped\")' <<< '${{ toJson(needs) }}'\necho \"status=$?\" >> $GITHUB_OUTPUT\n" + run: jq --exit-status 'all(.result == "success" or .result == "skipped")' <<< '${{ toJson(needs) }}' + # Publish the toolstate if an auto build succeeds (just before push to master) - name: publish toolstate run: src/ci/publish_toolstate.sh shell: bash - if: "steps.outputs.status == 0 && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'" + if: success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust' env: - TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" - - name: set the correct exit status - run: "exit ${{ steps.outputs.status == 0 }}" + TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }} diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml deleted file mode 100644 index 4ca7b52bc02..00000000000 --- a/src/ci/github-actions/ci.yml +++ /dev/null @@ -1,301 +0,0 @@ -###################################################### -# WARNING! Action needed when changing this file # -###################################################### - -# Due to GitHub Actions limitations, we can't use YAML Anchors directly in the -# CI configuration stored on the repository. To work around that this file is -# expanded by a tool in the repository, and the expansion is committed as well. -# -# After you make any change to the file you'll need to run this command: -# -# ./x.py run src/tools/expand-yaml-anchors -# -# ...and commit the file it updated in addition to this one. If you forget this -# step CI will fail. - ---- -############################### -# YAML Anchors Definition # -############################### - -# This key contains most of the YAML anchors that will be used later in the -# document. YAML anchors allows us to greatly reduce duplication inside the CI -# configuration by reusing parts of the configuration. -# -# YAML anchors work by defining an anchor with `&anchor-name` and reusing its -# content in another place with `*anchor-name`. The special `<<` map key merges -# the content of the map with the content of the anchor (or list of anchors). -# -# The expand-yaml-anchors tool will automatically remove this block from the -# output YAML file. -x--expand-yaml-anchors--remove: - # These snippets are used by the try-success, try-failure, auto-success and auto-failure jobs. - # Check out their documentation for more information on why they're needed. - - - &base-outcome-job - name: bors build finished - runs-on: ubuntu-latest - - - &base-success-job - steps: - - name: mark the job as a success - run: exit 0 - shell: bash - <<: *base-outcome-job - - - &base-failure-job - steps: - - name: mark the job as a failure - run: exit 1 - shell: bash - <<: *base-outcome-job - -########################### -# Builders definition # -########################### - -name: CI -on: - push: - branches: - - auto - - try - - try-perf - - automation/bors/try - pull_request: - branches: - - "**" - -permissions: - contents: read - packages: write - -defaults: - run: - # On Linux, macOS, and Windows, use the system-provided bash as the default - # shell. (This should only make a difference on Windows, where the default - # shell is PowerShell.) - shell: bash - -concurrency: - # For a given workflow, if we push to the same branch, cancel all previous builds on that branch. - # We add an exception for try builds (try branch) and unrolled rollup builds (try-perf), which - # are all triggered on the same branch, but which should be able to run concurrently. - group: ${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }} - cancel-in-progress: true - -env: - TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate - -jobs: - # The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml. - # It calculates which jobs should be executed, based on the data of the ${{ github }} context. - # If you want to modify CI jobs, take a look at src/ci/github-actions/jobs.yml. - calculate_matrix: - name: Calculate job matrix - runs-on: ubuntu-latest - outputs: - jobs: ${{ steps.jobs.outputs.jobs }} - steps: - - name: Checkout the source code - uses: actions/checkout@v4 - - name: Calculate the CI job matrix - run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT - id: jobs - job: - name: ${{ matrix.name }} - needs: [ calculate_matrix ] - runs-on: "${{ matrix.os }}" - defaults: - run: - shell: ${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }} - timeout-minutes: 600 - env: - CI_JOB_NAME: ${{ matrix.image }} - CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse - # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs. - HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SCCACHE_BUCKET: rust-lang-ci-sccache2 - CACHE_DOMAIN: ci-caches.rust-lang.org - continue-on-error: ${{ matrix.continue_on_error || false }} - strategy: - matrix: - # Check the `calculate_matrix` job to see how is the matrix defined. - include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }} - # GitHub Actions fails the workflow if an empty list of jobs is provided to - # the workflow, so we need to skip this job if nothing was produced by - # the Python script. - # - # Unfortunately checking whether a list is empty is not possible in a nice - # way due to GitHub Actions expressions limits. - # This hack is taken from https://github.com/ferrocene/ferrocene/blob/d43edc6b7697cf1719ec1c17c54904ab94825763/.github/workflows/release.yml#L75-L82 - if: fromJSON(needs.calculate_matrix.outputs.jobs)[0] != null - steps: - - if: contains(matrix.os, 'windows') - uses: msys2/setup-msys2@v2.22.0 - with: - # i686 jobs use mingw32. x86_64 and cross-compile jobs use mingw64. - msystem: ${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }} - # don't try to download updates for already installed packages - update: false - # don't try to use the msys that comes built-in to the github runner, - # so we can control what is installed (i.e. not python) - release: true - # Inherit the full path from the Windows environment, with MSYS2's */bin/ - # dirs placed in front. This lets us run Windows-native Python etc. - path-type: inherit - install: > - make - dos2unix - diffutils - - - name: disable git crlf conversion - run: git config --global core.autocrlf false - - - name: checkout the source code - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - # Rust Log Analyzer can't currently detect the PR number of a GitHub - # Actions build on its own, so a hint in the log message is needed to - # point it in the right direction. - - name: configure the PR in which the error message will be posted - run: echo "[CI_PR_NUMBER=$num]" - env: - num: ${{ github.event.number }} - if: success() && github.event_name == 'pull_request' - - - name: add extra environment variables - run: src/ci/scripts/setup-environment.sh - env: - # Since it's not possible to merge `${{ matrix.env }}` with the other - # variables in `job..env`, the variables defined in the matrix - # are passed to the `setup-environment.sh` script encoded in JSON, - # which then uses log commands to actually set them. - EXTRA_VARIABLES: ${{ toJson(matrix.env) }} - - - name: ensure the channel matches the target branch - run: src/ci/scripts/verify-channel.sh - - - name: collect CPU statistics - run: src/ci/scripts/collect-cpu-stats.sh - - - name: show the current environment - run: src/ci/scripts/dump-environment.sh - - - name: install awscli - run: src/ci/scripts/install-awscli.sh - - - name: install sccache - run: src/ci/scripts/install-sccache.sh - - - name: select Xcode - run: src/ci/scripts/select-xcode.sh - - - name: install clang - run: src/ci/scripts/install-clang.sh - - - name: install tidy - run: src/ci/scripts/install-tidy.sh - - - name: install WIX - run: src/ci/scripts/install-wix.sh - - - name: disable git crlf conversion - run: src/ci/scripts/disable-git-crlf-conversion.sh - - - name: checkout submodules - run: src/ci/scripts/checkout-submodules.sh - - - name: install MSYS2 - run: src/ci/scripts/install-msys2.sh - - - name: install MinGW - run: src/ci/scripts/install-mingw.sh - - - name: install ninja - run: src/ci/scripts/install-ninja.sh - - - name: enable ipv6 on Docker - run: src/ci/scripts/enable-docker-ipv6.sh - - # Disable automatic line ending conversion (again). On Windows, when we're - # installing dependencies, something switches the git configuration directory or - # re-enables autocrlf. We've not tracked down the exact cause -- and there may - # be multiple -- but this should ensure submodules are checked out with the - # appropriate line endings. - - name: disable git crlf conversion - run: src/ci/scripts/disable-git-crlf-conversion.sh - - - name: ensure line endings are correct - run: src/ci/scripts/verify-line-endings.sh - - - name: ensure backported commits are in upstream branches - run: src/ci/scripts/verify-backported-commits.sh - - - name: ensure the stable version number is correct - run: src/ci/scripts/verify-stable-version-number.sh - - - name: run the build - # Redirect stderr to stdout to avoid reordering the two streams in the GHA logs. - run: src/ci/scripts/run-build-from-ci.sh 2>&1 - env: - AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }} - TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }} - - - name: create github artifacts - run: src/ci/scripts/create-doc-artifacts.sh - - - name: upload artifacts to github - uses: actions/upload-artifact@v4 - with: - # name is set in previous step - name: ${{ env.DOC_ARTIFACT_NAME }} - path: obj/artifacts/doc - if-no-files-found: ignore - retention-days: 5 - - - name: upload artifacts to S3 - run: src/ci/scripts/upload-artifacts.sh - env: - AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }} - # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy - # builders *should* have the AWS credentials available. Still, explicitly - # adding the condition is helpful as this way CI will not silently skip - # deploying artifacts from a dist builder if the variables are misconfigured, - # erroring about invalid credentials instead. - if: success() && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1') - - # This job isused to tell bors the final status of the build, as there is no practical way to detect - # when a workflow is successful listening to webhooks only in our current bors implementation (homu). - outcome: - name: bors build finished - runs-on: ubuntu-latest - needs: [ job ] - # !cancelled() executes the job regardless of whether the previous jobs passed or failed - if: "!cancelled() && github.event_name == 'push'" - steps: - - name: checkout the source code - uses: actions/checkout@v4 - with: - fetch-depth: 2 - # Calculate the exit status of the whole CI workflow (0 if all dependent jobs were either successful - # or skipped, otherwise 1). - - name: calculate the correct exit status - id: status - run: | - jq --exit-status 'all(.result == "success" or .result == "skipped")' <<< '${{ toJson(needs) }}' - echo "status=$?" >> $GITHUB_OUTPUT - # Publish the toolstate if an auto build succeeds (just before push to master) - - name: publish toolstate - run: src/ci/publish_toolstate.sh - shell: bash - if: steps.outputs.status == 0 && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust' - env: - TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }} - - name: set the correct exit status - run: exit ${{ steps.outputs.status }}