mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Enable downloading prebuilt LLVM in test builders

Browse Source 
See comment added for details on the test builder restriction. This is primarily
intended for macOS CI, but is likely to be a slight win on other builders too.
This commit is contained in:
Mark Rousskov 2022-07-31 17:45:15 -04:00
parent 8ca14aa7ff
commit 6c7f301145
16 changed files with 88 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/ci.yml vendored
View File

@ -391,24 +391,28 @@ jobs:
env:
RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --set llvm.allow-old-toolchain"
SCRIPT: make ci-mingw-subset-1
NO_DOWNLOAD_CI_LLVM: 1
CUSTOM_MINGW: 1
os: windows-latest-xl
- name: i686-mingw-2
env:
RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --set llvm.allow-old-toolchain"
SCRIPT: make ci-mingw-subset-2
NO_DOWNLOAD_CI_LLVM: 1
CUSTOM_MINGW: 1
os: windows-latest-xl
- name: x86_64-mingw-1
env:
SCRIPT: make ci-mingw-subset-1
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-profiler --set llvm.allow-old-toolchain"
NO_DOWNLOAD_CI_LLVM: 1
CUSTOM_MINGW: 1
os: windows-latest-xl
- name: x86_64-mingw-2
env:
SCRIPT: make ci-mingw-subset-2
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-profiler --set llvm.allow-old-toolchain"
NO_DOWNLOAD_CI_LLVM: 1
CUSTOM_MINGW: 1
os: windows-latest-xl
- name: dist-x86_64-msvc
@ -433,6 +437,7 @@ jobs:
- name: dist-i686-mingw
env:
RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --enable-full-tools --enable-profiler --set llvm.allow-old-toolchain"
NO_DOWNLOAD_CI_LLVM: 1
SCRIPT: python x.py dist
CUSTOM_MINGW: 1
DIST_REQUIRE_ALL_TOOLS: 1
@ -441,6 +446,7 @@ jobs:
env:
SCRIPT: python x.py dist
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler --set llvm.allow-old-toolchain"
NO_DOWNLOAD_CI_LLVM: 1
CUSTOM_MINGW: 1
DIST_REQUIRE_ALL_TOOLS: 1
os: windows-latest-xl

2
src/bootstrap/config.rs
View File

@ -990,7 +990,7 @@ impl Config {
config.llvm_from_ci = match llvm.download_ci_llvm {
Some(StringOrBool::String(s)) => {
assert!(s == "if-available", "unknown option `{}` for download-ci-llvm", s);
crate::native::is_ci_llvm_available(&config)
crate::native::is_ci_llvm_available(&config, llvm_assertions.unwrap_or(false))
}
Some(StringOrBool::Bool(b)) => b,
None => false,

8
src/bootstrap/native.rs
View File

@ -148,7 +148,7 @@ pub(crate) fn detect_llvm_sha(config: &crate::config::Config) -> String {
/// This checks both the build triple platform to confirm we're usable at all,
/// and then verifies if the current HEAD matches the detected LLVM SHA head,
/// in which case LLVM is indicated as not available.
pub(crate) fn is_ci_llvm_available(config: &crate::config::Config) -> bool {
pub(crate) fn is_ci_llvm_available(config: &crate::config::Config, asserts: bool) -> bool {
// This is currently all tier 1 targets and tier 2 targets with host tools
// (since others may not have CI artifacts)
// https://doc.rust-lang.org/rustc/platform-support.html#tier-1
@ -187,6 +187,12 @@ pub(crate) fn is_ci_llvm_available(config: &crate::config::Config) -> bool {
return false;
}
let triple = &*config.build.triple;
if (triple == "aarch64-unknown-linux-gnu" || triple.contains("i686")) && asserts {
// No alt builder for aarch64-unknown-linux-gnu today.
return false;
}
if crate::util::CiEnv::is_ci() {
let llvm_sha = detect_llvm_sha(config);
let head_sha = output(config.git().arg("rev-parse").arg("HEAD"));

4
src/ci/docker/host-x86_64/arm-android/Dockerfile
View File

@ -29,6 +29,10 @@ ENV PATH=$PATH:/android/sdk/platform-tools
ENV TARGETS=arm-linux-androideabi
# We are intentionally allowing an old toolchain on this builder (and that's
# incompatible with LLVM downloads today).
ENV NO_DOWNLOAD_CI_LLVM 1
ENV RUST_CONFIGURE_ARGS --arm-linux-androideabi-ndk=/android/ndk/arm-14 \
--set llvm.allow-old-toolchain

4
src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile
View File

@ -27,6 +27,10 @@ RUN mkdir -p /config
RUN echo "[rust]" > /config/nopt-std-config.toml
RUN echo "optimize = false" >> /config/nopt-std-config.toml
# We are intentionally allowing an old toolchain on this builder (and that's
# incompatible with LLVM downloads today).
ENV NO_DOWNLOAD_CI_LLVM 1
ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu --disable-optimize-tests \
--set llvm.allow-old-toolchain
ENV SCRIPT python3 ../x.py test --stage 0 --config /config/nopt-std-config.toml library/std \

3
src/ci/docker/host-x86_64/i686-gnu/Dockerfile
View File

@ -23,6 +23,9 @@ RUN sh /scripts/sccache.sh
COPY scripts/cmake.sh /scripts/
RUN /scripts/cmake.sh
# We are intentionally allowing an old toolchain on this builder (and that's
# incompatible with LLVM downloads today).
ENV NO_DOWNLOAD_CI_LLVM 1
ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu \
--set llvm.allow-old-toolchain
# Exclude some tests that are unlikely to be platform specific, to speed up

4
src/ci/docker/host-x86_64/test-various/Dockerfile
View File

@ -30,6 +30,10 @@ WORKDIR /
COPY scripts/sccache.sh /scripts/
RUN sh /scripts/sccache.sh
# We are disabling CI LLVM since this builder needs to build LLD, which is
# currently unsupported when downloading pre-built LLVM.
ENV NO_DOWNLOAD_CI_LLVM 1
ENV RUST_CONFIGURE_ARGS \
--musl-root-x86_64=/usr/local/x86_64-linux-musl \
--set build.nodejs=/node-v15.14.0-linux-x64/bin/node \

4
src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile
View File

@ -26,6 +26,10 @@ RUN sh /scripts/sccache.sh
COPY scripts/cmake.sh /scripts/
RUN /scripts/cmake.sh
# We are intentionally allowing an old toolchain on this builder (and that's
# incompatible with LLVM downloads today).
ENV NO_DOWNLOAD_CI_LLVM 1
ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu \
--set llvm.allow-old-toolchain
ENV RUST_CHECK_TARGET check-aux

3
src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile
View File

@ -31,6 +31,9 @@ RUN sh /scripts/sccache.sh
ENV RUSTBUILD_FORCE_CLANG_BASED_TESTS 1
ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1
# llvm.use-linker conflicts with downloading CI LLVM
ENV NO_DOWNLOAD_CI_LLVM 1
ENV RUST_CONFIGURE_ARGS \
--build=x86_64-unknown-linux-gnu \
--enable-debug \

4
src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile
View File

@ -22,6 +22,10 @@ RUN sh /scripts/sccache.sh
COPY scripts/cmake.sh /scripts/
RUN /scripts/cmake.sh
# We are intentionally allowing an old toolchain on this builder (and that's
# incompatible with LLVM downloads today).
ENV NO_DOWNLOAD_CI_LLVM 1
ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu --set rust.ignore-git=false \
--set llvm.allow-old-toolchain
ENV SCRIPT python3 ../x.py --stage 2 test distcheck

6
src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile
View File

@ -26,7 +26,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
COPY scripts/sccache.sh /scripts/
RUN sh /scripts/sccache.sh
# using llvm-link-shared due to libffi issues -- see #34486
# We are disabling CI LLVM since this builder is intentionally using a host
# LLVM, rather than the typical src/llvm-project LLVM.
ENV NO_DOWNLOAD_CI_LLVM 1
# Using llvm-link-shared due to libffi issues -- see #34486
ENV RUST_CONFIGURE_ARGS \
--build=x86_64-unknown-linux-gnu \
--llvm-root=/usr/lib/llvm-13 \

6
src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile
View File

@ -37,7 +37,11 @@ RUN apt-get update && \
COPY scripts/sccache.sh /scripts/
RUN sh /scripts/sccache.sh
# using llvm-link-shared due to libffi issues -- see #34486
# We are disabling CI LLVM since this builder is intentionally using a host
# LLVM, rather than the typical src/llvm-project LLVM.
ENV NO_DOWNLOAD_CI_LLVM 1
# Using llvm-link-shared due to libffi issues -- see #34486
ENV RUST_CONFIGURE_ARGS \
--build=x86_64-unknown-linux-gnu \
--llvm-root=/usr/lib/llvm-13 \

4
src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile
View File

@ -80,6 +80,10 @@ COPY host-x86_64/x86_64-gnu-tools/browser-ui-test.version /tmp/
# the local version of the package is different than the one used by the CI.
RUN npm install -g browser-ui-test@$(head -n 1 /tmp/browser-ui-test.version) --unsafe-perm=true
# We are intentionally allowing an old toolchain on this builder (and that's
# incompatible with LLVM downloads today).
ENV NO_DOWNLOAD_CI_LLVM 1
ENV RUST_CONFIGURE_ARGS \
--set llvm.allow-old-toolchain \
--build=x86_64-unknown-linux-gnu \

18
src/ci/github-actions/ci.yml
View File

@ -600,6 +600,9 @@ jobs:
--build=i686-pc-windows-gnu
--set llvm.allow-old-toolchain
SCRIPT: make ci-mingw-subset-1
# We are intentionally allowing an old toolchain on this builder (and that's
# incompatible with LLVM downloads today).
NO_DOWNLOAD_CI_LLVM: 1
CUSTOM_MINGW: 1
<<: *job-windows-xl
@ -609,6 +612,9 @@ jobs:
--build=i686-pc-windows-gnu
--set llvm.allow-old-toolchain
SCRIPT: make ci-mingw-subset-2
# We are intentionally allowing an old toolchain on this builder (and that's
# incompatible with LLVM downloads today).
NO_DOWNLOAD_CI_LLVM: 1
CUSTOM_MINGW: 1
<<: *job-windows-xl
@ -619,6 +625,9 @@ jobs:
--build=x86_64-pc-windows-gnu
--enable-profiler
--set llvm.allow-old-toolchain
# We are intentionally allowing an old toolchain on this builder (and that's
# incompatible with LLVM downloads today).
NO_DOWNLOAD_CI_LLVM: 1
CUSTOM_MINGW: 1
<<: *job-windows-xl
@ -629,6 +638,9 @@ jobs:
--build=x86_64-pc-windows-gnu
--enable-profiler
--set llvm.allow-old-toolchain
# We are intentionally allowing an old toolchain on this builder (and that's
# incompatible with LLVM downloads today).
NO_DOWNLOAD_CI_LLVM: 1
CUSTOM_MINGW: 1
<<: *job-windows-xl
@ -678,6 +690,9 @@ jobs:
--enable-full-tools
--enable-profiler
--set llvm.allow-old-toolchain
# We are intentionally allowing an old toolchain on this builder (and that's
# incompatible with LLVM downloads today).
NO_DOWNLOAD_CI_LLVM: 1
SCRIPT: python x.py dist
CUSTOM_MINGW: 1
DIST_REQUIRE_ALL_TOOLS: 1
@ -691,6 +706,9 @@ jobs:
--enable-full-tools
--enable-profiler
--set llvm.allow-old-toolchain
# We are intentionally allowing an old toolchain on this builder (and that's
# incompatible with LLVM downloads today).
NO_DOWNLOAD_CI_LLVM: 1
CUSTOM_MINGW: 1
DIST_REQUIRE_ALL_TOOLS: 1
<<: *job-windows-xl

12
src/ci/run.sh
View File

@ -102,6 +102,18 @@ else
fi
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir"
# We enable this for non-dist builders, since those aren't trying to produce
# fresh binaries. We currently don't entirely support distributing a fresh
# copy of the compiler (including llvm tools, etc.) if we haven't actually
# built LLVM, since not everything necessary is copied into the
# local-usage-only LLVM artifacts. If that changes, this could maybe be made
# true for all builds. In practice it's probably a good idea to keep building
# LLVM continuously on at least some builders to ensure it works, though.
# (And PGO is its own can of worms).
if [ "$NO_DOWNLOAD_CI_LLVM" = "" ]; then
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set llvm.download-ci-llvm=if-available"
fi
fi
if [ "$RUST_RELEASE_CHANNEL" = "nightly" ] || [ "$DIST_REQUIRE_ALL_TOOLS" = "" ]; then

4
src/ci/scripts/install-clang.sh
View File

@ -61,6 +61,10 @@ elif isWindows && [[ ${CUSTOM_MINGW-0} -ne 1 ]]; then
7z x -oclang-rust/ "LLVM-${LLVM_VERSION}-win64.exe"
ciCommandSetEnv RUST_CONFIGURE_ARGS \
"${RUST_CONFIGURE_ARGS} --set llvm.clang-cl=$(pwd)/clang-rust/bin/clang-cl.exe"
# Disable downloading CI LLVM on this builder;
# setting up clang-cl just above conflicts with the default if-available option.
ciCommandSetEnv NO_DOWNLOAD_CI_LLVM 1
fi
if isWindows; then