From 71e50180713eaaaa3bedc827e6890cede03baa0b Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 30 Oct 2019 12:26:02 +0100 Subject: [PATCH 1/9] ci: replace MINGW_URL with CUSTOM_MINGW=1 This commit replaces the mirrors base URL contained in the MINGW_URL with a CUSTOM_MINGW=1 environment variable. The mirrors base URL will be fetched instead through the MIRRORS_BASE environment variable, defined in src/ci/shared.sh. --- src/ci/azure-pipelines/auto.yml | 12 ++++++------ src/ci/scripts/install-clang.sh | 2 +- src/ci/scripts/install-mingw.sh | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml index 836f81a7816..f69808c3702 100644 --- a/src/ci/azure-pipelines/auto.yml +++ b/src/ci/azure-pipelines/auto.yml @@ -284,7 +284,7 @@ jobs: MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make ci-mingw-subset-1 - MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc + CUSTOM_MINGW: 1 MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z MINGW_DIR: mingw32 # FIXME(#59637) @@ -294,14 +294,14 @@ jobs: MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make ci-mingw-subset-2 - MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc + CUSTOM_MINGW: 1 MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z MINGW_DIR: mingw32 x86_64-mingw-1: MSYS_BITS: 64 SCRIPT: make ci-mingw-subset-1 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu - MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc + CUSTOM_MINGW: 1 MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z MINGW_DIR: mingw64 # FIXME(#59637) @@ -311,7 +311,7 @@ jobs: MSYS_BITS: 64 SCRIPT: make ci-mingw-subset-2 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu - MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc + CUSTOM_MINGW: 1 MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z MINGW_DIR: mingw64 @@ -340,7 +340,7 @@ jobs: MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler SCRIPT: python x.py dist - MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc + CUSTOM_MINGW: 1 MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z MINGW_DIR: mingw32 DIST_REQUIRE_ALL_TOOLS: 1 @@ -349,7 +349,7 @@ jobs: MSYS_BITS: 64 SCRIPT: python x.py dist RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler - MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc + CUSTOM_MINGW: 1 MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z MINGW_DIR: mingw64 DIST_REQUIRE_ALL_TOOLS: 1 diff --git a/src/ci/scripts/install-clang.sh b/src/ci/scripts/install-clang.sh index b1e9bf92ca5..f0179994e8e 100755 --- a/src/ci/scripts/install-clang.sh +++ b/src/ci/scripts/install-clang.sh @@ -17,7 +17,7 @@ if isMacOS; then # Configure `AR` specifically so rustbuild doesn't try to infer it as # `clang-ar` by accident. ciCommandSetEnv AR "ar" -elif isWindows && [[ -z ${MINGW_URL+x} ]]; then +elif isWindows && [[ ${CUSTOM_MINGW-0} -ne 1 ]]; then # If we're compiling for MSVC then we, like most other distribution builders, # switch to clang as the compiler. This'll allow us eventually to enable LTO # amongst LLVM and rustc. Note that we only do this on MSVC as I don't think diff --git a/src/ci/scripts/install-mingw.sh b/src/ci/scripts/install-mingw.sh index b4e8b889f52..8b11e59fb63 100755 --- a/src/ci/scripts/install-mingw.sh +++ b/src/ci/scripts/install-mingw.sh @@ -28,7 +28,7 @@ IFS=$'\n\t' source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" if isWindows; then - if [[ -z "${MINGW_URL+x}" ]]; then + if [[ "${CUSTOM_MINGW-0}" -ne 1 ]]; then arch=i686 if [ "$MSYS_BITS" = "64" ]; then arch=x86_64 @@ -37,9 +37,9 @@ if isWindows; then mingw-w64-$arch-gcc mingw-w64-$arch-python2 ciCommandAddPath "${SYSTEM_WORKFOLDER}/msys2/mingw${MSYS_BITS}/bin" else - curl -o mingw.7z "${MINGW_URL}/${MINGW_ARCHIVE}" + curl -o mingw.7z "${MIRRORS_BASE}/${MINGW_ARCHIVE}" 7z x -y mingw.7z > /dev/null - curl -o "${MINGW_DIR}/bin/gdborig.exe" "${MINGW_URL}/2017-04-20-${MSYS_BITS}bit-gdborig.exe" + curl -o "${MINGW_DIR}/bin/gdborig.exe" "${MIRRORS_BASE}/2017-04-20-${MSYS_BITS}bit-gdborig.exe" ciCommandAddPath "$(pwd)/${MINGW_DIR}/bin" fi fi From 53c2c04d63b3c043ec5b1b4f401e04868a1139a4 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 30 Oct 2019 12:35:55 +0100 Subject: [PATCH 2/9] ci: remove the MINGW_DIR and MINGW_ARCHIVE env vars --- src/ci/azure-pipelines/auto.yml | 12 ------------ src/ci/scripts/install-mingw.sh | 15 ++++++++++++--- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml index f69808c3702..74fb237eebd 100644 --- a/src/ci/azure-pipelines/auto.yml +++ b/src/ci/azure-pipelines/auto.yml @@ -285,8 +285,6 @@ jobs: RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make ci-mingw-subset-1 CUSTOM_MINGW: 1 - MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - MINGW_DIR: mingw32 # FIXME(#59637) NO_DEBUG_ASSERTIONS: 1 NO_LLVM_ASSERTIONS: 1 @@ -295,15 +293,11 @@ jobs: RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make ci-mingw-subset-2 CUSTOM_MINGW: 1 - MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - MINGW_DIR: mingw32 x86_64-mingw-1: MSYS_BITS: 64 SCRIPT: make ci-mingw-subset-1 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu CUSTOM_MINGW: 1 - MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - MINGW_DIR: mingw64 # FIXME(#59637) NO_DEBUG_ASSERTIONS: 1 NO_LLVM_ASSERTIONS: 1 @@ -312,8 +306,6 @@ jobs: SCRIPT: make ci-mingw-subset-2 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu CUSTOM_MINGW: 1 - MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - MINGW_DIR: mingw64 # 32/64 bit MSVC and GNU deployment dist-x86_64-msvc: @@ -341,8 +333,6 @@ jobs: RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler SCRIPT: python x.py dist CUSTOM_MINGW: 1 - MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - MINGW_DIR: mingw32 DIST_REQUIRE_ALL_TOOLS: 1 DEPLOY: 1 dist-x86_64-mingw: @@ -350,8 +340,6 @@ jobs: SCRIPT: python x.py dist RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler CUSTOM_MINGW: 1 - MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - MINGW_DIR: mingw64 DIST_REQUIRE_ALL_TOOLS: 1 DEPLOY: 1 diff --git a/src/ci/scripts/install-mingw.sh b/src/ci/scripts/install-mingw.sh index 8b11e59fb63..631d5e575a5 100755 --- a/src/ci/scripts/install-mingw.sh +++ b/src/ci/scripts/install-mingw.sh @@ -27,6 +27,9 @@ IFS=$'\n\t' source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" +MINGW_ARCHIVE_32="i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z" +MINGW_ARCHIVE_64="x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z" + if isWindows; then if [[ "${CUSTOM_MINGW-0}" -ne 1 ]]; then arch=i686 @@ -37,9 +40,15 @@ if isWindows; then mingw-w64-$arch-gcc mingw-w64-$arch-python2 ciCommandAddPath "${SYSTEM_WORKFOLDER}/msys2/mingw${MSYS_BITS}/bin" else - curl -o mingw.7z "${MIRRORS_BASE}/${MINGW_ARCHIVE}" + mingw_archive="${MINGW_ARCHIVE_32}" + if [[ "${MSYS_BITS}" = "64" ]]; then + mingw_archive="${MINGW_ARCHIVE_64}" + fi + mingw_dir="mingw${MSYS_BITS}" + + curl -o mingw.7z "${MIRRORS_BASE}/${mingw_archive}" 7z x -y mingw.7z > /dev/null - curl -o "${MINGW_DIR}/bin/gdborig.exe" "${MIRRORS_BASE}/2017-04-20-${MSYS_BITS}bit-gdborig.exe" - ciCommandAddPath "$(pwd)/${MINGW_DIR}/bin" + curl -o "${mingw_dir}/bin/gdborig.exe" "${MIRRORS_BASE}/2017-04-20-${MSYS_BITS}bit-gdborig.exe" + ciCommandAddPath "$(pwd)/${mingw_dir}/bin" fi fi From af6b26646b41e1b9614be12ae126f7033a3fa908 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 30 Oct 2019 12:45:03 +0100 Subject: [PATCH 3/9] ci: remove the MSYS_BITS env var --- src/ci/azure-pipelines/auto.yml | 17 --------------- src/ci/azure-pipelines/steps/run.yml | 14 ++++++------ src/ci/azure-pipelines/try.yml | 1 - src/ci/scripts/install-mingw.sh | 32 ++++++++++++++++++---------- 4 files changed, 28 insertions(+), 36 deletions(-) diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml index 74fb237eebd..946eb483c29 100644 --- a/src/ci/azure-pipelines/auto.yml +++ b/src/ci/azure-pipelines/auto.yml @@ -223,25 +223,21 @@ jobs: matrix: # 32/64 bit MSVC tests x86_64-msvc-1: - MSYS_BITS: 64 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler SCRIPT: make ci-subset-1 # FIXME(#59637) NO_DEBUG_ASSERTIONS: 1 NO_LLVM_ASSERTIONS: 1 x86_64-msvc-2: - MSYS_BITS: 64 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler SCRIPT: make ci-subset-2 i686-msvc-1: - MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc SCRIPT: make ci-subset-1 # FIXME(#59637) NO_DEBUG_ASSERTIONS: 1 NO_LLVM_ASSERTIONS: 1 i686-msvc-2: - MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc SCRIPT: make ci-subset-2 # FIXME(#59637) @@ -249,11 +245,9 @@ jobs: NO_LLVM_ASSERTIONS: 1 # MSVC aux tests x86_64-msvc-aux: - MSYS_BITS: 64 RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc x86_64-msvc-cargo: - MSYS_BITS: 64 SCRIPT: python x.py test src/tools/cargotest src/tools/cargo RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc VCVARS_BAT: vcvars64.bat @@ -262,10 +256,8 @@ jobs: NO_LLVM_ASSERTIONS: 1 # MSVC tools tests x86_64-msvc-tools: - MSYS_BITS: 64 SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json - DEPLOY_TOOLSTATES_JSON: toolstates-windows.json # 32/64-bit MinGW builds. # @@ -281,7 +273,6 @@ jobs: # came from the mingw-w64 SourceForge download site. Unfortunately # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. i686-mingw-1: - MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make ci-mingw-subset-1 CUSTOM_MINGW: 1 @@ -289,12 +280,10 @@ jobs: NO_DEBUG_ASSERTIONS: 1 NO_LLVM_ASSERTIONS: 1 i686-mingw-2: - MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make ci-mingw-subset-2 CUSTOM_MINGW: 1 x86_64-mingw-1: - MSYS_BITS: 64 SCRIPT: make ci-mingw-subset-1 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu CUSTOM_MINGW: 1 @@ -302,14 +291,12 @@ jobs: NO_DEBUG_ASSERTIONS: 1 NO_LLVM_ASSERTIONS: 1 x86_64-mingw-2: - MSYS_BITS: 64 SCRIPT: make ci-mingw-subset-2 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu CUSTOM_MINGW: 1 # 32/64 bit MSVC and GNU deployment dist-x86_64-msvc: - MSYS_BITS: 64 RUST_CONFIGURE_ARGS: >- --build=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc @@ -319,7 +306,6 @@ jobs: DIST_REQUIRE_ALL_TOOLS: 1 DEPLOY: 1 dist-i686-msvc: - MSYS_BITS: 32 RUST_CONFIGURE_ARGS: >- --build=i686-pc-windows-msvc --target=i586-pc-windows-msvc @@ -329,14 +315,12 @@ jobs: DIST_REQUIRE_ALL_TOOLS: 1 DEPLOY: 1 dist-i686-mingw: - MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler SCRIPT: python x.py dist CUSTOM_MINGW: 1 DIST_REQUIRE_ALL_TOOLS: 1 DEPLOY: 1 dist-x86_64-mingw: - MSYS_BITS: 64 SCRIPT: python x.py dist RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler CUSTOM_MINGW: 1 @@ -345,7 +329,6 @@ jobs: # "alternate" deployment, see .travis.yml for more info dist-x86_64-msvc-alt: - MSYS_BITS: 64 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler SCRIPT: python x.py dist DEPLOY_ALT: 1 diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index cef2d235602..4d7c6859322 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -8,6 +8,13 @@ steps: +# Configure our CI_JOB_NAME variable which log analyzers can use for the main +# step to see what's going on. +- bash: | + builder=$(echo $AGENT_JOBNAME | cut -d ' ' -f 2) + echo "##vso[task.setvariable variable=CI_JOB_NAME]$builder" + displayName: Configure Job Name + # Disable automatic line ending conversion, which is enabled by default on # Azure's Windows image. Having the conversion enabled caused regressions both # in our test suite (it broke miri tests) and in the ecosystem, since we @@ -135,13 +142,6 @@ steps: condition: and(succeeded(), not(variables.SKIP_JOB)) displayName: Install awscli -# Configure our CI_JOB_NAME variable which log analyzers can use for the main -# step to see what's going on. -- bash: | - builder=$(echo $AGENT_JOBNAME | cut -d ' ' -f 2) - echo "##vso[task.setvariable variable=CI_JOB_NAME]$builder" - displayName: Configure Job Name - # As a quick smoke check on the otherwise very fast mingw-check linux builder # check our own internal scripts. - bash: | diff --git a/src/ci/azure-pipelines/try.yml b/src/ci/azure-pipelines/try.yml index c919b1023a0..fe39ce3e241 100644 --- a/src/ci/azure-pipelines/try.yml +++ b/src/ci/azure-pipelines/try.yml @@ -72,7 +72,6 @@ jobs: # DEPLOY: 1 # # dist-x86_64-msvc-alt: -# MSYS_BITS: 64 # RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler # SCRIPT: python x.py dist # DEPLOY_ALT: 1 diff --git a/src/ci/scripts/install-mingw.sh b/src/ci/scripts/install-mingw.sh index 631d5e575a5..8b579587b9e 100755 --- a/src/ci/scripts/install-mingw.sh +++ b/src/ci/scripts/install-mingw.sh @@ -31,24 +31,34 @@ MINGW_ARCHIVE_32="i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z" MINGW_ARCHIVE_64="x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z" if isWindows; then + case "${CI_JOB_NAME}" in + *i686*) + bits=32 + arch=i686 + mingw_archive="${MINGW_ARCHIVE_32}" + ;; + *x86_64*) + bits=64 + arch=x86_64 + mingw_archive="${MINGW_ARCHIVE_64}" + ;; + *) + echo "src/ci/scripts/install-mingw.sh can't detect the builder's architecture" + echo "please tweak it to recognize the builder named '${CI_JOB_NAME}'" + exit 1 + ;; + esac + if [[ "${CUSTOM_MINGW-0}" -ne 1 ]]; then - arch=i686 - if [ "$MSYS_BITS" = "64" ]; then - arch=x86_64 - fi pacman -S --noconfirm --needed mingw-w64-$arch-toolchain mingw-w64-$arch-cmake \ mingw-w64-$arch-gcc mingw-w64-$arch-python2 - ciCommandAddPath "${SYSTEM_WORKFOLDER}/msys2/mingw${MSYS_BITS}/bin" + ciCommandAddPath "${SYSTEM_WORKFOLDER}/msys2/mingw${bits}/bin" else - mingw_archive="${MINGW_ARCHIVE_32}" - if [[ "${MSYS_BITS}" = "64" ]]; then - mingw_archive="${MINGW_ARCHIVE_64}" - fi - mingw_dir="mingw${MSYS_BITS}" + mingw_dir="mingw${bits}" curl -o mingw.7z "${MIRRORS_BASE}/${mingw_archive}" 7z x -y mingw.7z > /dev/null - curl -o "${mingw_dir}/bin/gdborig.exe" "${MIRRORS_BASE}/2017-04-20-${MSYS_BITS}bit-gdborig.exe" + curl -o "${mingw_dir}/bin/gdborig.exe" "${MIRRORS_BASE}/2017-04-20-${bits}bit-gdborig.exe" ciCommandAddPath "$(pwd)/${mingw_dir}/bin" fi fi From 6104aa77eb97050c3b302a0d13934040de84cda9 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 30 Oct 2019 18:13:18 +0100 Subject: [PATCH 4/9] ci: extract validate-toolstate into a script --- src/ci/azure-pipelines/steps/run.yml | 13 +------------ src/ci/scripts/validate-toolstate.sh | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 12 deletions(-) create mode 100755 src/ci/scripts/validate-toolstate.sh diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index 4d7c6859322..4df315d5d5c 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -144,18 +144,7 @@ steps: # As a quick smoke check on the otherwise very fast mingw-check linux builder # check our own internal scripts. -- bash: | - set -e - git clone --depth=1 https://github.com/rust-lang-nursery/rust-toolstate.git - cd rust-toolstate - python2.7 "$BUILD_SOURCESDIRECTORY/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "" "" - # Only check maintainers if this build is supposed to publish toolstate. - # Builds that are not supposed to publish don't have the access token. - if [ -n "${TOOLSTATE_PUBLISH+is_set}" ]; then - TOOLSTATE_VALIDATE_MAINTAINERS_REPO=rust-lang/rust python2.7 "${BUILD_SOURCESDIRECTORY}/src/tools/publish_toolstate.py" - fi - cd .. - rm -rf rust-toolstate +- bash: src/ci/scripts/validate-toolstate.sh env: TOOLSTATE_REPO_ACCESS_TOKEN: $(TOOLSTATE_REPO_ACCESS_TOKEN) condition: and(succeeded(), not(variables.SKIP_JOB), eq(variables['IMAGE'], 'mingw-check')) diff --git a/src/ci/scripts/validate-toolstate.sh b/src/ci/scripts/validate-toolstate.sh new file mode 100755 index 00000000000..93e645b24a8 --- /dev/null +++ b/src/ci/scripts/validate-toolstate.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# A quick smoke test to make sure publish_tooolstate.py works. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +git clone --depth=1 https://github.com/rust-lang-nursery/rust-toolstate.git +cd rust-toolstate +python2.7 "${BUILD_SOURCESDIRECTORY}/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "" "" +# Only check maintainers if this build is supposed to publish toolstate. +# Builds that are not supposed to publish don't have the access token. +if [ -n "${TOOLSTATE_PUBLISH+is_set}" ]; then + TOOLSTATE_VALIDATE_MAINTAINERS_REPO=rust-lang/rust python2.7 "${BUILD_SOURCESDIRECTORY}/src/tools/publish_toolstate.py" +fi +cd .. +rm -rf rust-toolstate From d623c56f4c2eab00a9cb5ba12e9e708430945f1a Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 30 Oct 2019 18:18:49 +0100 Subject: [PATCH 5/9] ci: extract running the build into a script --- src/ci/azure-pipelines/steps/run.yml | 15 +-------------- src/ci/scripts/run-build-from-ci.sh | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 14 deletions(-) create mode 100755 src/ci/scripts/run-build-from-ci.sh diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index 4df315d5d5c..fe2bee50715 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -150,22 +150,9 @@ steps: condition: and(succeeded(), not(variables.SKIP_JOB), eq(variables['IMAGE'], 'mingw-check')) displayName: Verify the publish_toolstate script works -- bash: | - set -e - # Remove any preexisting rustup installation since it can interfere - # with the cargotest step and its auto-detection of things like Clippy in - # the environment - rustup self uninstall -y || true - if [ "$IMAGE" = "" ]; then - src/ci/run.sh - else - src/ci/docker/run.sh $IMAGE - fi - #timeoutInMinutes: 180 +- bash: src/ci/scripts/run-build-from-ci.sh timeoutInMinutes: 600 env: - CI: true - SRC: . AWS_ACCESS_KEY_ID: $(SCCACHE_AWS_ACCESS_KEY_ID) AWS_SECRET_ACCESS_KEY: $(SCCACHE_AWS_SECRET_ACCESS_KEY) TOOLSTATE_REPO_ACCESS_TOKEN: $(TOOLSTATE_REPO_ACCESS_TOKEN) diff --git a/src/ci/scripts/run-build-from-ci.sh b/src/ci/scripts/run-build-from-ci.sh new file mode 100755 index 00000000000..c02117f459d --- /dev/null +++ b/src/ci/scripts/run-build-from-ci.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Start the CI build. You shouldn't run this locally: call either src/ci/run.sh +# or src/ci/docker/run.sh instead. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +export CI="true" +export SRC=. + +# Remove any preexisting rustup installation since it can interfere +# with the cargotest step and its auto-detection of things like Clippy in +# the environment +rustup self uninstall -y || true +if [ -z "${IMAGE+x}" ]; then + src/ci/run.sh +else + src/ci/docker/run.sh "${IMAGE}" +fi From e209ee42e99cd76d7fefd0c53fbd84f0d2123935 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 30 Oct 2019 18:56:40 +0100 Subject: [PATCH 6/9] ci: extract collecting cpu stats into a script --- src/ci/azure-pipelines/steps/run.yml | 7 ++----- src/ci/scripts/collect-cpu-stats.sh | 9 +++++++++ 2 files changed, 11 insertions(+), 5 deletions(-) create mode 100755 src/ci/scripts/collect-cpu-stats.sh diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index fe2bee50715..e856207b17d 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -31,11 +31,8 @@ steps: - bash: src/ci/scripts/should-skip-this.sh displayName: Decide whether to run this job -# Spawn a background process to collect CPU usage statistics which we'll upload -# at the end of the build. See the comments in the script here for more -# information. -- bash: python src/ci/cpu-usage-over-time.py &> cpu-usage.csv & - displayName: "Collect CPU-usage statistics in the background" +- bash: src/ci/scripts/collect-cpu-stats.sh + displayName: Collect CPU-usage statistics in the background - bash: src/ci/scripts/dump-environment.sh displayName: Show the current environment diff --git a/src/ci/scripts/collect-cpu-stats.sh b/src/ci/scripts/collect-cpu-stats.sh new file mode 100755 index 00000000000..08065431f98 --- /dev/null +++ b/src/ci/scripts/collect-cpu-stats.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Spawn a background process to collect CPU usage statistics which we'll upload +# at the end of the build. See the comments in the script here for more +# information. + +set -euo pipefail +IFS=$'\n\t' + +python src/ci/cpu-usage-over-time.py &> cpu-usage.csv & From c90cc12b0721cc6e80ba62041bf0b82058428580 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Thu, 31 Oct 2019 16:56:11 +0100 Subject: [PATCH 7/9] ci: move validate-toolstate.sh in the mingw-check image The task was already run just there, so this cleans things up. --- src/ci/azure-pipelines/steps/run.yml | 8 -------- src/ci/docker/mingw-check/Dockerfile | 5 ++++- .../mingw-check}/validate-toolstate.sh | 9 +++++---- 3 files changed, 9 insertions(+), 13 deletions(-) rename src/ci/{scripts => docker/mingw-check}/validate-toolstate.sh (57%) diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index e856207b17d..214c11fd690 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -139,14 +139,6 @@ steps: condition: and(succeeded(), not(variables.SKIP_JOB)) displayName: Install awscli -# As a quick smoke check on the otherwise very fast mingw-check linux builder -# check our own internal scripts. -- bash: src/ci/scripts/validate-toolstate.sh - env: - TOOLSTATE_REPO_ACCESS_TOKEN: $(TOOLSTATE_REPO_ACCESS_TOKEN) - condition: and(succeeded(), not(variables.SKIP_JOB), eq(variables['IMAGE'], 'mingw-check')) - displayName: Verify the publish_toolstate script works - - bash: src/ci/scripts/run-build-from-ci.sh timeoutInMinutes: 600 env: diff --git a/src/ci/docker/mingw-check/Dockerfile b/src/ci/docker/mingw-check/Dockerfile index 24e2dea4ca7..b2d96aed2a9 100644 --- a/src/ci/docker/mingw-check/Dockerfile +++ b/src/ci/docker/mingw-check/Dockerfile @@ -19,7 +19,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh +COPY mingw-check/validate-toolstate.sh /scripts/ + ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1 ENV SCRIPT python2.7 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \ python2.7 ../x.py build --stage 0 src/tools/build-manifest && \ - python2.7 ../x.py test --stage 0 src/tools/compiletest + python2.7 ../x.py test --stage 0 src/tools/compiletest && \ + /scripts/validate-toolstate.sh diff --git a/src/ci/scripts/validate-toolstate.sh b/src/ci/docker/mingw-check/validate-toolstate.sh similarity index 57% rename from src/ci/scripts/validate-toolstate.sh rename to src/ci/docker/mingw-check/validate-toolstate.sh index 93e645b24a8..2ebf1d6d5ae 100755 --- a/src/ci/scripts/validate-toolstate.sh +++ b/src/ci/docker/mingw-check/validate-toolstate.sh @@ -4,15 +4,16 @@ set -euo pipefail IFS=$'\n\t' -source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" - +rm -rf rust-toolstate git clone --depth=1 https://github.com/rust-lang-nursery/rust-toolstate.git cd rust-toolstate -python2.7 "${BUILD_SOURCESDIRECTORY}/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "" "" +python2.7 "../../src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" \ + "$(git log --format=%s -n1 HEAD)" "" "" # Only check maintainers if this build is supposed to publish toolstate. # Builds that are not supposed to publish don't have the access token. if [ -n "${TOOLSTATE_PUBLISH+is_set}" ]; then - TOOLSTATE_VALIDATE_MAINTAINERS_REPO=rust-lang/rust python2.7 "${BUILD_SOURCESDIRECTORY}/src/tools/publish_toolstate.py" + TOOLSTATE_VALIDATE_MAINTAINERS_REPO=rust-lang/rust python2.7 \ + "../../src/tools/publish_toolstate.py" fi cd .. rm -rf rust-toolstate From 14da85c68e6b5cd8a64a1f2f5ba5cd50c725e38b Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 4 Nov 2019 17:40:18 +0100 Subject: [PATCH 8/9] ci: move mirrors to https://ci-mirrors.rust-lang.org --- src/ci/docker/armhf-gnu/Dockerfile | 2 +- src/ci/docker/dist-armv7-linux/crosstool-ng.sh | 2 +- src/ci/docker/dist-various-1/install-mips-musl.sh | 2 +- src/ci/docker/dist-various-1/install-mipsel-musl.sh | 2 +- src/ci/docker/dist-various-2/build-wasi-toolchain.sh | 2 +- src/ci/docker/dist-x86_64-linux/build-curl.sh | 2 +- src/ci/docker/dist-x86_64-linux/build-openssl.sh | 2 +- src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh | 2 +- src/ci/docker/scripts/freebsd-toolchain.sh | 2 +- src/ci/docker/scripts/sccache.sh | 2 +- src/ci/shared.sh | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/ci/docker/armhf-gnu/Dockerfile b/src/ci/docker/armhf-gnu/Dockerfile index 9493b336987..5373612279b 100644 --- a/src/ci/docker/armhf-gnu/Dockerfile +++ b/src/ci/docker/armhf-gnu/Dockerfile @@ -72,7 +72,7 @@ RUN arm-linux-gnueabihf-gcc addentropy.c -o rootfs/addentropy -static # TODO: What is this?! # Source of the file: https://github.com/vfdev-5/qemu-rpi2-vexpress/raw/master/vexpress-v2p-ca15-tc1.dtb -RUN curl -O https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/vexpress-v2p-ca15-tc1.dtb +RUN curl -O https://ci-mirrors.rust-lang.org/rustc/vexpress-v2p-ca15-tc1.dtb COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh diff --git a/src/ci/docker/dist-armv7-linux/crosstool-ng.sh b/src/ci/docker/dist-armv7-linux/crosstool-ng.sh index ae737d9677d..fb067a79a5c 100644 --- a/src/ci/docker/dist-armv7-linux/crosstool-ng.sh +++ b/src/ci/docker/dist-armv7-linux/crosstool-ng.sh @@ -1,7 +1,7 @@ set -ex # Mirrored from https://github.com/crosstool-ng/crosstool-ng/archive/crosstool-ng-1.24.0.tar.gz -url="https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/crosstool-ng-1.24.0.tar.gz" +url="https://ci-mirrors.rust-lang.org/rustc/crosstool-ng-1.24.0.tar.gz" curl -Lf $url | tar xzf - cd crosstool-ng-crosstool-ng-1.24.0 ./bootstrap diff --git a/src/ci/docker/dist-various-1/install-mips-musl.sh b/src/ci/docker/dist-various-1/install-mips-musl.sh index 29cfb5d9608..9584258d234 100755 --- a/src/ci/docker/dist-various-1/install-mips-musl.sh +++ b/src/ci/docker/dist-various-1/install-mips-musl.sh @@ -5,7 +5,7 @@ mkdir /usr/local/mips-linux-musl # originally from # https://downloads.openwrt.org/snapshots/trunk/ar71xx/generic/ # OpenWrt-Toolchain-ar71xx-generic_gcc-5.3.0_musl-1.1.16.Linux-x86_64.tar.bz2 -URL="https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc" +URL="https://ci-mirrors.rust-lang.org/rustc" FILE="OpenWrt-Toolchain-ar71xx-generic_gcc-5.3.0_musl-1.1.16.Linux-x86_64.tar.bz2" curl -L "$URL/$FILE" | tar xjf - -C /usr/local/mips-linux-musl --strip-components=2 diff --git a/src/ci/docker/dist-various-1/install-mipsel-musl.sh b/src/ci/docker/dist-various-1/install-mipsel-musl.sh index de8c359d167..50a8e554b16 100755 --- a/src/ci/docker/dist-various-1/install-mipsel-musl.sh +++ b/src/ci/docker/dist-various-1/install-mipsel-musl.sh @@ -5,7 +5,7 @@ mkdir /usr/local/mipsel-linux-musl # Note that this originally came from: # https://downloads.openwrt.org/snapshots/trunk/malta/generic/ # OpenWrt-Toolchain-malta-le_gcc-5.3.0_musl-1.1.15.Linux-x86_64.tar.bz2 -URL="https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc" +URL="https://ci-mirrors.rust-lang.org/rustc" FILE="OpenWrt-Toolchain-malta-le_gcc-5.3.0_musl-1.1.15.Linux-x86_64.tar.bz2" curl -L "$URL/$FILE" | tar xjf - -C /usr/local/mipsel-linux-musl --strip-components=2 diff --git a/src/ci/docker/dist-various-2/build-wasi-toolchain.sh b/src/ci/docker/dist-various-2/build-wasi-toolchain.sh index c63ea6facca..17aa78945cf 100755 --- a/src/ci/docker/dist-various-2/build-wasi-toolchain.sh +++ b/src/ci/docker/dist-various-2/build-wasi-toolchain.sh @@ -5,7 +5,7 @@ set -ex # Originally from https://releases.llvm.org/9.0.0/clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz -curl https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/clang%2Bllvm-9.0.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz | \ +curl https://ci-mirrors.rust-lang.org/rustc/clang%2Bllvm-9.0.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz | \ tar xJf - export PATH=`pwd`/clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-14.04/bin:$PATH diff --git a/src/ci/docker/dist-x86_64-linux/build-curl.sh b/src/ci/docker/dist-x86_64-linux/build-curl.sh index 8200bbe2fdc..a65e3de5cfc 100755 --- a/src/ci/docker/dist-x86_64-linux/build-curl.sh +++ b/src/ci/docker/dist-x86_64-linux/build-curl.sh @@ -5,7 +5,7 @@ source shared.sh VERSION=7.66.0 -curl https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/curl-$VERSION.tar.xz \ +curl https://ci-mirrors.rust-lang.org/rustc/curl-$VERSION.tar.xz \ | xz --decompress \ | tar xf - diff --git a/src/ci/docker/dist-x86_64-linux/build-openssl.sh b/src/ci/docker/dist-x86_64-linux/build-openssl.sh index be8a6c93945..0d9bdf7a099 100755 --- a/src/ci/docker/dist-x86_64-linux/build-openssl.sh +++ b/src/ci/docker/dist-x86_64-linux/build-openssl.sh @@ -4,7 +4,7 @@ set -ex source shared.sh VERSION=1.0.2k -URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/openssl-$VERSION.tar.gz +URL=https://ci-mirrors.rust-lang.org/rustc/openssl-$VERSION.tar.gz curl $URL | tar xzf - diff --git a/src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh b/src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh index 797f674b954..f8697c698b9 100755 --- a/src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh +++ b/src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh @@ -25,7 +25,7 @@ cd netbsd mkdir -p /x-tools/x86_64-unknown-netbsd/sysroot -URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc +URL=https://ci-mirrors.rust-lang.org/rustc # Originally from ftp://ftp.netbsd.org/pub/NetBSD/NetBSD-$BSD/source/sets/*.tgz curl $URL/2018-03-01-netbsd-src.tgz | tar xzf - diff --git a/src/ci/docker/scripts/freebsd-toolchain.sh b/src/ci/docker/scripts/freebsd-toolchain.sh index 70155e770a9..5670e10be23 100755 --- a/src/ci/docker/scripts/freebsd-toolchain.sh +++ b/src/ci/docker/scripts/freebsd-toolchain.sh @@ -59,7 +59,7 @@ done # Originally downloaded from: # https://download.freebsd.org/ftp/releases/${freebsd_arch}/${freebsd_version}-RELEASE/base.txz -URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2019-04-04-freebsd-${freebsd_arch}-${freebsd_version}-RELEASE-base.txz +URL=https://ci-mirrors.rust-lang.org/rustc/2019-04-04-freebsd-${freebsd_arch}-${freebsd_version}-RELEASE-base.txz curl "$URL" | tar xJf - -C "$sysroot" --wildcards "${files_to_extract[@]}" # Fix up absolute symlinks from the system image. This can be removed diff --git a/src/ci/docker/scripts/sccache.sh b/src/ci/docker/scripts/sccache.sh index efeb0ed0d72..552afbfee7c 100644 --- a/src/ci/docker/scripts/sccache.sh +++ b/src/ci/docker/scripts/sccache.sh @@ -1,6 +1,6 @@ set -ex curl -fo /usr/local/bin/sccache \ - https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2018-04-02-sccache-x86_64-unknown-linux-musl + https://ci-mirrors.rust-lang.org/rustc/2018-04-02-sccache-x86_64-unknown-linux-musl chmod +x /usr/local/bin/sccache diff --git a/src/ci/shared.sh b/src/ci/shared.sh index 718a5379ae5..862ded0d5db 100644 --- a/src/ci/shared.sh +++ b/src/ci/shared.sh @@ -4,7 +4,7 @@ # `source shared.sh`, hence the invalid shebang and not being # marked as an executable file in git. -export MIRRORS_BASE="https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc" +export MIRRORS_BASE="https://ci-mirrors.rust-lang.org/rustc" # See http://unix.stackexchange.com/questions/82598 # Duplicated in docker/dist-various-2/shared.sh From 85132b20c4b3f6858ea273ec276c4ad9162603f8 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 6 Nov 2019 18:58:57 +0100 Subject: [PATCH 9/9] ci: download curl and openssl from s3 for dist-x86_64-linux CentOS 5 only supports SSLv3 without SNI, and to get newer protocols working we need to download and compile OpenSSL and cURL from our mirror. Because of that, we can't use the CDN, as CloudFront requires TLSv1 with SNI. This commit changes the dist-x86_64-linux image to bypass the CDN for OpenSSL and cURL. --- src/ci/docker/dist-x86_64-linux/build-curl.sh | 5 ++++- src/ci/docker/dist-x86_64-linux/build-openssl.sh | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/ci/docker/dist-x86_64-linux/build-curl.sh b/src/ci/docker/dist-x86_64-linux/build-curl.sh index a65e3de5cfc..0c7eb5fdac9 100755 --- a/src/ci/docker/dist-x86_64-linux/build-curl.sh +++ b/src/ci/docker/dist-x86_64-linux/build-curl.sh @@ -5,7 +5,10 @@ source shared.sh VERSION=7.66.0 -curl https://ci-mirrors.rust-lang.org/rustc/curl-$VERSION.tar.xz \ +# This needs to be downloaded directly from S3, it can't go through the CDN. +# That's because the CDN is backed by CloudFront, which requires SNI and TLSv1 +# (without paying an absurd amount of money). +curl https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/curl-$VERSION.tar.xz \ | xz --decompress \ | tar xf - diff --git a/src/ci/docker/dist-x86_64-linux/build-openssl.sh b/src/ci/docker/dist-x86_64-linux/build-openssl.sh index 0d9bdf7a099..d8f6bdb51b8 100755 --- a/src/ci/docker/dist-x86_64-linux/build-openssl.sh +++ b/src/ci/docker/dist-x86_64-linux/build-openssl.sh @@ -4,7 +4,11 @@ set -ex source shared.sh VERSION=1.0.2k -URL=https://ci-mirrors.rust-lang.org/rustc/openssl-$VERSION.tar.gz + +# This needs to be downloaded directly from S3, it can't go through the CDN. +# That's because the CDN is backed by CloudFront, which requires SNI and TLSv1 +# (without paying an absurd amount of money). +URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/openssl-$VERSION.tar.gz curl $URL | tar xzf -