From 80459c14a4a6d9613d2bc07f5f0df905e8f99a0d Mon Sep 17 00:00:00 2001 From: Matt Harding Date: Tue, 13 Feb 2024 04:37:31 +0000 Subject: [PATCH] Changes to CI related to mingw and MSYS --- .github/workflows/ci.yml | 33 ++++++++++++++++++++++++++++ src/ci/github-actions/ci.yml | 21 ++++++++++++++++++ src/ci/run.sh | 2 +- src/ci/scripts/install-clang.sh | 3 +-- src/ci/scripts/install-mingw.sh | 10 ++++----- src/ci/scripts/install-msys2.sh | 38 +++++++++++++++++++++++++++------ src/ci/shared.sh | 4 ++++ 7 files changed, 96 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 26e589c092e..464fd3b5640 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,9 +65,20 @@ jobs: - name: x86_64-gnu-tools os: ubuntu-20.04-16core-64gb env: {} + defaults: + run: + shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}" timeout-minutes: 600 runs-on: "${{ matrix.os }}" steps: + - if: "contains(matrix.os, 'windows')" + uses: msys2/setup-msys2@v2.22.0 + with: + msystem: "${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}" + update: false + release: true + path-type: inherit + install: "make dos2unix diffutils\n" - name: disable git crlf conversion run: git config --global core.autocrlf false - name: checkout the source code @@ -459,9 +470,20 @@ jobs: RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-extended --enable-profiler" SCRIPT: python x.py dist bootstrap --include-default-paths os: windows-2019-8core-32gb + defaults: + run: + shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}" timeout-minutes: 600 runs-on: "${{ matrix.os }}" steps: + - if: "contains(matrix.os, 'windows')" + uses: msys2/setup-msys2@v2.22.0 + with: + msystem: "${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}" + update: false + release: true + path-type: inherit + install: "make dos2unix diffutils\n" - name: disable git crlf conversion run: git config --global core.autocrlf false - name: checkout the source code @@ -587,9 +609,20 @@ jobs: env: CODEGEN_BACKENDS: "llvm,cranelift" os: ubuntu-20.04-16core-64gb + defaults: + run: + shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}" timeout-minutes: 600 runs-on: "${{ matrix.os }}" steps: + - if: "contains(matrix.os, 'windows')" + uses: msys2/setup-msys2@v2.22.0 + with: + msystem: "${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}" + update: false + release: true + path-type: inherit + install: "make dos2unix diffutils\n" - name: disable git crlf conversion run: git config --global core.autocrlf false - name: checkout the source code diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 43e48c01176..51d9dea5a0b 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -111,10 +111,31 @@ x--expand-yaml-anchors--remove: if: success() && !env.SKIP_JOB - &base-ci-job + defaults: + run: + shell: ${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }} timeout-minutes: 600 runs-on: "${{ matrix.os }}" env: *shared-ci-variables steps: + - if: contains(matrix.os, 'windows') + uses: msys2/setup-msys2@v2.22.0 + with: + # i686 jobs use mingw32. x86_64 and cross-compile jobs use mingw64. + msystem: ${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }} + # don't try to download updates for already installed packages + update: false + # don't try to use the msys that comes built-in to the github runner, + # so we can control what is installed (i.e. not python) + release: true + # Inherit the full path from the Windows environment, with MSYS2's */bin/ + # dirs placed in front. This lets us run Windows-native Python etc. + path-type: inherit + install: > + make + dos2unix + diffutils + - name: disable git crlf conversion run: git config --global core.autocrlf false diff --git a/src/ci/run.sh b/src/ci/run.sh index 1cdcffc1a75..3ad04c73d3d 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -76,7 +76,7 @@ RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set dist.compression-profile=balance # the LLVM build, as not to run out of memory. # This is an attempt to fix the spurious build error tracked by # https://github.com/rust-lang/rust/issues/108227. -if isWindows && [[ ${CUSTOM_MINGW-0} -eq 1 ]]; then +if isKnownToBeMingwBuild; then RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set llvm.link-jobs=1" fi diff --git a/src/ci/scripts/install-clang.sh b/src/ci/scripts/install-clang.sh index 77164ed4117..aa7ff813f51 100755 --- a/src/ci/scripts/install-clang.sh +++ b/src/ci/scripts/install-clang.sh @@ -37,8 +37,7 @@ if isMacOS; then # Configure `AR` specifically so rustbuild doesn't try to infer it as # `clang-ar` by accident. ciCommandSetEnv AR "ar" -elif isWindows && [[ ${CUSTOM_MINGW-0} -ne 1 ]]; then - +elif isWindows && ! isKnownToBeMingwBuild; then # If we're compiling for MSVC then we, like most other distribution builders, # switch to clang as the compiler. This'll allow us eventually to enable LTO # amongst LLVM and rustc. Note that we only do this on MSVC as I don't think diff --git a/src/ci/scripts/install-mingw.sh b/src/ci/scripts/install-mingw.sh index 7eccb9b8650..87b835b63db 100755 --- a/src/ci/scripts/install-mingw.sh +++ b/src/ci/scripts/install-mingw.sh @@ -38,11 +38,11 @@ if isWindows; then ;; esac - if [[ "${CUSTOM_MINGW-0}" -ne 1 ]]; then - pacman -S --noconfirm --needed mingw-w64-$arch-toolchain mingw-w64-$arch-cmake \ - mingw-w64-$arch-gcc \ - mingw-w64-$arch-python # the python package is actually for python3 - ciCommandAddPath "$(ciCheckoutPath)/msys2/mingw${bits}/bin" + if [[ "${CUSTOM_MINGW:-0}" == 0 ]]; then + pacboy -S --noconfirm toolchain:p + # According to the comment in the Windows part of install-clang.sh, in the future we might + # want to do this instead: + # pacboy -S --noconfirm clang:p ... else mingw_dir="mingw${bits}" diff --git a/src/ci/scripts/install-msys2.sh b/src/ci/scripts/install-msys2.sh index 0aa4b42a6a8..905edf38a09 100755 --- a/src/ci/scripts/install-msys2.sh +++ b/src/ci/scripts/install-msys2.sh @@ -1,17 +1,12 @@ #!/bin/bash -# Download and install MSYS2, needed primarily for the test suite (run-make) but -# also used by the MinGW toolchain for assembling things. +# Clean up and prepare the MSYS2 installation. MSYS2 is needed primarily for +# the test suite (run-make), but is also used by the MinGW toolchain for assembling things. set -euo pipefail IFS=$'\n\t' source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" - if isWindows; then - msys2Path="c:/msys64" - mkdir -p "${msys2Path}/home/${USERNAME}" - ciCommandAddPath "${msys2Path}/usr/bin" - # Detect the native Python version installed on the agent. On GitHub # Actions, the C:\hostedtoolcache\windows\Python directory contains a # subdirectory for each installed Python version. @@ -29,4 +24,33 @@ if isWindows; then fi ciCommandAddPath "C:\\hostedtoolcache\\windows\\Python\\${native_python_version}\\x64" ciCommandAddPath "C:\\hostedtoolcache\\windows\\Python\\${native_python_version}\\x64\\Scripts" + + # Install pacboy for easily installing packages + pacman -S --noconfirm pactoys + + # Delete these pre-installed tools so we can't accidentally use them, because we are using the + # MSYS2 setup action versions instead. + # Delete pre-installed version of MSYS2 + rm -r "/c/msys64/" + # Delete Strawberry Perl, which contains a version of mingw + rm -r "/c/Strawberry/" + # Delete these other copies of mingw, I don't even know where they come from. + rm -r "/c/mingw64/" + rm -r "/c/mingw32/" + + if isKnownToBeMingwBuild; then + # Use the mingw version of CMake for mingw builds. + # However, the MSVC build needs native CMake, as it fails with the mingw one. + # Delete native CMake + rm -r "/c/Program Files/CMake/" + # Install mingw-w64-$arch-cmake + pacboy -S --noconfirm cmake:p + + # We use Git-for-Windows for MSVC builds, and MSYS2 Git for mingw builds, + # so that both are tested. + # Delete Windows-Git + rm -r "/c/Program Files/Git/" + # Install MSYS2 git + pacman -S --noconfirm git + fi fi diff --git a/src/ci/shared.sh b/src/ci/shared.sh index 720394af249..2b0a10e4d08 100644 --- a/src/ci/shared.sh +++ b/src/ci/shared.sh @@ -52,6 +52,10 @@ function isLinux { [[ "${OSTYPE}" = "linux-gnu" ]] } +function isKnownToBeMingwBuild { + isGitHubActions && [[ "${CI_JOB_NAME}" == *mingw ]] +} + function isCiBranch { if [[ $# -ne 1 ]]; then echo "usage: $0 "