From c22c81cbc1eebff1521a3cdefceb4c287fa8dfe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Thu, 4 Apr 2024 14:27:38 +0200 Subject: [PATCH 1/3] Generate CI job matrix for PR jobs in Python --- .github/workflows/ci.yml | 28 ++++++++-------- src/ci/github-actions/ci.yml | 27 ++++++++------- src/ci/github-actions/jobs.yml | 46 ++++++++++++++++++++++++++ src/ci/scripts/calculate-job-matrix.py | 25 ++++++++++++++ 4 files changed, 98 insertions(+), 28 deletions(-) create mode 100644 src/ci/github-actions/jobs.yml create mode 100755 src/ci/scripts/calculate-job-matrix.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1c87b0a76e..cfeb6b985ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,8 +36,21 @@ concurrency: group: "${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }}" cancel-in-progress: true jobs: + calculate_matrix: + name: Calculate job matrix + runs-on: ubuntu-latest + outputs: + jobs: "${{ steps.jobs.outputs.jobs }}" + steps: + - name: Checkout the source code + uses: actions/checkout@v4 + - name: Calculate the CI job matrix + run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT + id: jobs pr: name: "PR - ${{ matrix.name }}" + needs: + - calculate_matrix env: PR_CI_JOB: 1 CI_JOB_NAME: "${{ matrix.name }}" @@ -51,20 +64,7 @@ jobs: continue-on-error: "${{ matrix.name == 'mingw-check-tidy' }}" strategy: matrix: - include: - - name: mingw-check - os: ubuntu-20.04-4core-16gb - env: {} - - name: mingw-check-tidy - os: ubuntu-20.04-4core-16gb - env: {} - - name: x86_64-gnu-llvm-17 - env: - ENABLE_GCC_CODEGEN: "1" - os: ubuntu-20.04-16core-64gb - - name: x86_64-gnu-tools - os: ubuntu-20.04-16core-64gb - env: {} + include: "${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}" defaults: run: shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}" diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 80e23574404..8f8b10ab90e 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -340,9 +340,21 @@ concurrency: cancel-in-progress: true jobs: + calculate_matrix: + name: Calculate job matrix + runs-on: ubuntu-latest + outputs: + jobs: ${{ steps.jobs.outputs.jobs }} + steps: + - name: Checkout the source code + uses: actions/checkout@v4 + - name: Calculate the CI job matrix + run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT + id: jobs pr: <<: *base-ci-job name: PR - ${{ matrix.name }} + needs: [ calculate_matrix ] env: <<: [*shared-ci-variables, *public-variables] PR_CI_JOB: 1 @@ -350,20 +362,7 @@ jobs: continue-on-error: ${{ matrix.name == 'mingw-check-tidy' }} strategy: matrix: - include: - - name: mingw-check - <<: *job-linux-4c - - - name: mingw-check-tidy - <<: *job-linux-4c - - - name: x86_64-gnu-llvm-17 - env: - ENABLE_GCC_CODEGEN: "1" - <<: *job-linux-16c - - - name: x86_64-gnu-tools - <<: *job-linux-16c + include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }} auto: <<: *base-ci-job diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml new file mode 100644 index 00000000000..5b0140053d6 --- /dev/null +++ b/src/ci/github-actions/jobs.yml @@ -0,0 +1,46 @@ +x--expand-yaml-anchors--remove: + - &base-job + env: { } + + - &job-linux-4c + os: ubuntu-20.04-4core-16gb + <<: *base-job + + - &job-linux-8c + os: ubuntu-20.04-8core-32gb + <<: *base-job + + - &job-linux-16c + os: ubuntu-20.04-16core-64gb + <<: *base-job + + - &job-macos-xl + os: macos-13 # We use the standard runner for now + <<: *base-job + + - &job-macos-m1 + os: macos-14 + <<: *base-job + + - &job-windows-8c + os: windows-2019-8core-32gb + <<: *base-job + + - &job-windows-16c + os: windows-2019-16core-64gb + <<: *base-job + + - &job-aarch64-linux + os: [ self-hosted, ARM64, linux ] + +pr: + - name: mingw-check + <<: *job-linux-4c + - name: mingw-check-tidy + <<: *job-linux-4c + - name: x86_64-gnu-llvm-17 + env: + ENABLE_GCC_CODEGEN: "1" + <<: *job-linux-16c + - name: x86_64-gnu-tools + <<: *job-linux-16c diff --git a/src/ci/scripts/calculate-job-matrix.py b/src/ci/scripts/calculate-job-matrix.py new file mode 100755 index 00000000000..9b1e74c23c3 --- /dev/null +++ b/src/ci/scripts/calculate-job-matrix.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 + +""" +This script serves for generating a matrix of jobs that should +be executed on CI. + +It reads job definitions from `src/ci/github-actions/jobs.yml` +and filters them based on the event that happened on CI. + +Currently, it only supports PR builds. +""" + +import json +from pathlib import Path + +import yaml + +JOBS_YAML_PATH = Path(__file__).absolute().parent.parent / "github-actions" / "jobs.yml" + + +if __name__ == "__main__": + with open(JOBS_YAML_PATH) as f: + jobs = yaml.safe_load(f) + job_output = jobs["pr"] + print(f"jobs={json.dumps(job_output)}") From f15b3267519488be17623322598d91b854a71227 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Thu, 4 Apr 2024 15:18:24 +0200 Subject: [PATCH 2/3] Select specific CI YAML files to expand --- src/tools/expand-yaml-anchors/src/main.rs | 37 +++++++++-------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/src/tools/expand-yaml-anchors/src/main.rs b/src/tools/expand-yaml-anchors/src/main.rs index 3fc72ecbbc4..60651734b9e 100644 --- a/src/tools/expand-yaml-anchors/src/main.rs +++ b/src/tools/expand-yaml-anchors/src/main.rs @@ -2,11 +2,11 @@ use std::path::{Path, PathBuf}; use yaml_rust::{Yaml, YamlEmitter, YamlLoader}; -/// List of directories containing files to expand. The first tuple element is the source -/// directory, while the second tuple element is the destination directory. +/// List of files to expand. The first tuple element is the source +/// file, while the second tuple element is the destination file. #[rustfmt::skip] static TO_EXPAND: &[(&str, &str)] = &[ - ("src/ci/github-actions", ".github/workflows"), + ("src/ci/github-actions/ci.yml", ".github/workflows/ci.yml"), ]; /// Name of a special key that will be removed from all the maps in expanded configuration files. @@ -62,27 +62,20 @@ fn from_args() -> Result> { fn run(&self) -> Result<(), Box> { for (source, dest) in TO_EXPAND { let source = self.base.join(source); - let dest = self.base.join(dest); - for entry in std::fs::read_dir(&source)? { - let path = entry?.path(); - if !path.is_file() || path.extension().and_then(|e| e.to_str()) != Some("yml") { - continue; - } + let dest_path = self.base.join(dest); - let dest_path = dest.join(path.file_name().unwrap()); - self.expand(&path, &dest_path).with_context(|| match self.mode { - Mode::Generate => format!( - "failed to expand {} into {}", - self.path(&path), - self.path(&dest_path) - ), - Mode::Check => format!( - "{} is not up to date; please run \ + self.expand(&source, &dest_path).with_context(|| match self.mode { + Mode::Generate => format!( + "failed to expand {} into {}", + self.path(&source), + self.path(&dest_path) + ), + Mode::Check => format!( + "{} is not up to date; please run \ `x.py run src/tools/expand-yaml-anchors`.", - self.path(&dest_path) - ), - })?; - } + self.path(&dest_path) + ), + })?; } Ok(()) } From 72104e245bf2cc4ce3e29cf28ebee95b78bcd786 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 15 Apr 2024 21:21:45 +0200 Subject: [PATCH 3/3] Add comments --- src/ci/github-actions/ci.yml | 2 ++ src/ci/github-actions/jobs.yml | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 8f8b10ab90e..ec65b5f37bd 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -340,6 +340,7 @@ concurrency: cancel-in-progress: true jobs: + # The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml. calculate_matrix: name: Calculate job matrix runs-on: ubuntu-latest @@ -362,6 +363,7 @@ jobs: continue-on-error: ${{ matrix.name == 'mingw-check-tidy' }} strategy: matrix: + # Check the `calculate_matrix` job to see how is the matrix defined. include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }} auto: diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index 5b0140053d6..7e89eef2670 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -1,4 +1,8 @@ -x--expand-yaml-anchors--remove: +# This file contains definitions of CI job parameters that are loaded +# dynamically in CI from ci.yml. +# You *do not* need to re-run `src/tools/expand-yaml-anchors` when you +# modify this file. +shared_defs: - &base-job env: { }