From a1b9a599584e9044da30bfd8fbc0d16ca72c009e Mon Sep 17 00:00:00 2001 From: Tyler Mandry Date: Tue, 7 Nov 2023 18:11:07 -0800 Subject: [PATCH] Build Fuchsia in CI --- .github/workflows/ci.yml | 3 + src/bootstrap/src/core/build_steps/compile.rs | 2 +- .../host-x86_64/dist-various-2/Dockerfile | 4 +- .../x86_64-gnu-integration/Dockerfile | 68 +++++++++++++++++++ .../x86_64-gnu-integration/build-fuchsia.sh | 51 ++++++++++++++ .../build-fuchsia-toolchain.sh | 8 +-- .../dist-various-2 => scripts}/shared.sh | 13 +++- src/ci/github-actions/ci.yml | 4 ++ src/ci/shared.sh | 2 +- 9 files changed, 144 insertions(+), 11 deletions(-) create mode 100644 src/ci/docker/host-x86_64/x86_64-gnu-integration/Dockerfile create mode 100755 src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh rename src/ci/docker/{host-x86_64/dist-various-2 => scripts}/build-fuchsia-toolchain.sh (85%) rename src/ci/docker/{host-x86_64/dist-various-2 => scripts}/shared.sh (68%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0d9eb8c452..4cf603519b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -288,6 +288,9 @@ jobs: - name: x86_64-gnu-aux os: ubuntu-20.04-4core-16gb env: {} + - name: x86_64-gnu-integration + os: ubuntu-20.04-16core-64gb + env: {} - name: x86_64-gnu-debug os: ubuntu-20.04-8core-32gb env: {} diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index c253dd2c6aa..1ab488ef656 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1722,7 +1722,7 @@ fn run(self, builder: &Builder<'_>) -> Compiler { let dst_exe = exe("rust-lld", target_compiler.host); builder.copy(&lld_install.join("bin").join(&src_exe), &libdir_bin.join(&dst_exe)); let self_contained_lld_dir = libdir_bin.join("gcc-ld"); - t!(fs::create_dir(&self_contained_lld_dir)); + t!(fs::create_dir_all(&self_contained_lld_dir)); let lld_wrapper_exe = builder.ensure(crate::core::build_steps::tool::LldWrapper { compiler: build_compiler, target: target_compiler.host, diff --git a/src/ci/docker/host-x86_64/dist-various-2/Dockerfile b/src/ci/docker/host-x86_64/dist-various-2/Dockerfile index 3f4e025111a..5f1fec74bed 100644 --- a/src/ci/docker/host-x86_64/dist-various-2/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-various-2/Dockerfile @@ -76,8 +76,8 @@ RUN env \ rm -rf /build/* WORKDIR /tmp -COPY host-x86_64/dist-various-2/shared.sh /tmp/ -COPY host-x86_64/dist-various-2/build-fuchsia-toolchain.sh /tmp/ +COPY scripts/shared.sh /tmp/ +COPY scripts/build-fuchsia-toolchain.sh /tmp/ RUN /tmp/build-fuchsia-toolchain.sh COPY host-x86_64/dist-various-2/build-solaris-toolchain.sh /tmp/ RUN /tmp/build-solaris-toolchain.sh x86_64 amd64 solaris-i386 pc diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-integration/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-integration/Dockerfile new file mode 100644 index 00000000000..ba65ba9bed4 --- /dev/null +++ b/src/ci/docker/host-x86_64/x86_64-gnu-integration/Dockerfile @@ -0,0 +1,68 @@ +FROM ubuntu:22.04 + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + g++ \ + make \ + ninja-build \ + file \ + curl \ + ca-certificates \ + python3 \ + git \ + cmake \ + libssl-dev \ + sudo \ + xz-utils \ + pkg-config \ + unzip \ + && rm -rf /var/lib/apt/lists/* + +# Duplicated in dist-various-2 Dockerfile. +# FIXME: Move to canonical triple +ENV \ + AR_x86_64_fuchsia=x86_64-unknown-fuchsia-ar \ + CC_x86_64_fuchsia=x86_64-unknown-fuchsia-clang \ + CFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \ + CXX_x86_64_fuchsia=x86_64-unknown-fuchsia-clang++ \ + CXXFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \ + LDFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -L/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/lib" + +WORKDIR /tmp +COPY scripts/shared.sh /tmp/ +COPY scripts/build-fuchsia-toolchain.sh /tmp/ +RUN /tmp/build-fuchsia-toolchain.sh + +ENV CARGO_TARGET_X86_64_FUCHSIA_AR /usr/local/bin/llvm-ar +ENV CARGO_TARGET_X86_64_FUCHSIA_RUSTFLAGS \ + -C panic=abort \ + -C force-unwind-tables=yes \ + -C link-arg=--sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot \ + -Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot/lib \ + -Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/lib + +ENV TARGETS=x86_64-fuchsia +ENV TARGETS=$TARGETS,x86_64-unknown-linux-gnu + +COPY scripts/sccache.sh /scripts/ +RUN sh /scripts/sccache.sh + +ENV RUST_INSTALL_DIR /checkout/obj/install +RUN mkdir -p $RUST_INSTALL_DIR/etc + +ENV RUST_CONFIGURE_ARGS \ + --prefix=$RUST_INSTALL_DIR \ + --sysconfdir=etc \ + --enable-lld \ + --llvm-libunwind=in-tree \ + --enable-extended \ + --disable-docs \ + --set target.x86_64-fuchsia.cc=/usr/local/bin/clang \ + --set target.x86_64-fuchsia.cxx=/usr/local/bin/clang++ \ + --set target.x86_64-fuchsia.ar=/usr/local/bin/llvm-ar \ + --set target.x86_64-fuchsia.ranlib=/usr/local/bin/llvm-ranlib \ + --set target.x86_64-fuchsia.linker=/usr/local/bin/ld.lld +ENV SCRIPT \ + python3 ../x.py install --target $TARGETS compiler/rustc library/std clippy && \ + bash ../src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh b/src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh new file mode 100755 index 00000000000..4a246f591d7 --- /dev/null +++ b/src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# Downloads and builds the Fuchsia operating system using a toolchain installed +# in $RUST_INSTALL_DIR. + +set -euf -o pipefail + +INTEGRATION_SHA=66793c4894bf6204579bbee3b79956335f31c768 +PICK_REFS=() + +checkout=fuchsia +jiri=.jiri_root/bin/jiri + +set -x + +# This script will: +# - create a directory named "fuchsia" if it does not exist +# - download "jiri" to "fuchsia/.jiri_root/bin" +curl -s "https://fuchsia.googlesource.com/jiri/+/HEAD/scripts/bootstrap_jiri?format=TEXT" \ + | base64 --decode \ + | bash -s $checkout + +cd $checkout + +$jiri init \ + -partial=true \ + -analytics-opt=false \ + . + +$jiri import \ + -name=integration \ + -revision=$INTEGRATION_SHA \ + -overwrite=true \ + flower \ + "https://fuchsia.googlesource.com/integration" + +if [ -d ".git" ]; then + # Wipe out any local changes if we're reusing a checkout. + git checkout --force JIRI_HEAD +fi + +$jiri update -autoupdate=false + +echo integration commit = $(git -C integration rev-parse HEAD) + +for git_ref in "${PICK_REFS[@]}"; do + git fetch https://fuchsia.googlesource.com/fuchsia $git_ref + git cherry-pick --no-commit FETCH_HEAD +done + +bash scripts/rust/build_fuchsia_from_rust_ci.sh diff --git a/src/ci/docker/host-x86_64/dist-various-2/build-fuchsia-toolchain.sh b/src/ci/docker/scripts/build-fuchsia-toolchain.sh similarity index 85% rename from src/ci/docker/host-x86_64/dist-various-2/build-fuchsia-toolchain.sh rename to src/ci/docker/scripts/build-fuchsia-toolchain.sh index d762b4672c6..beea2f522fd 100755 --- a/src/ci/docker/host-x86_64/dist-various-2/build-fuchsia-toolchain.sh +++ b/src/ci/docker/scripts/build-fuchsia-toolchain.sh @@ -4,13 +4,13 @@ set -ex source shared.sh FUCHSIA_SDK_URL=https://chrome-infra-packages.appspot.com/dl/fuchsia/sdk/core/linux-amd64 -FUCHSIA_SDK_ID=4xjxrGUrDbQ6_zJwj6cDN1IbWsWV5aCQXC_zO_Hu0XkC -FUCHSIA_SDK_SHA256=e318f1ac652b0db43aff32708fa70337521b5ac595e5a0905c2ff33bf1eed179 +FUCHSIA_SDK_ID=MrhQwtmP8CpZre-i_PNOREcThbUcrX3bA-45d6WQr-cC +FUCHSIA_SDK_SHA256=32b850c2d98ff02a59adefa2fcf34e44471385b51cad7ddb03ee3977a590afe7 FUCHSIA_SDK_USR_DIR=/usr/local/core-linux-amd64-fuchsia-sdk CLANG_DOWNLOAD_URL=\ https://chrome-infra-packages.appspot.com/dl/fuchsia/third_party/clang/linux-amd64 -CLANG_DOWNLOAD_ID=vU0vNjSihOV4Q6taQYCpy03JXGiCyVwxen3rFMNMIgsC -CLANG_DOWNLOAD_SHA256=bd4d2f3634a284e57843ab5a4180a9cb4dc95c6882c95c317a7deb14c34c220b +CLANG_DOWNLOAD_ID=Tpc85d1ZwSlZ6UKl2d96GRUBGNA5JKholOKe24sRDr0C +CLANG_DOWNLOAD_SHA256=4e973ce5dd59c12959e942a5d9df7a19150118d03924a86894e29edb8b110ebd install_clang() { mkdir -p clang_download diff --git a/src/ci/docker/host-x86_64/dist-various-2/shared.sh b/src/ci/docker/scripts/shared.sh similarity index 68% rename from src/ci/docker/host-x86_64/dist-various-2/shared.sh rename to src/ci/docker/scripts/shared.sh index 291f26bdaeb..9969659088d 100644 --- a/src/ci/docker/host-x86_64/dist-various-2/shared.sh +++ b/src/ci/docker/scripts/shared.sh @@ -1,5 +1,11 @@ -#!/usr/bin/env bash -hide_output() { +#!/bin/false +# shellcheck shell=bash + +# This file is intended to be sourced with `. shared.sh` or +# `source shared.sh`, hence the invalid shebang and not being +# marked as an executable file in git. + +function hide_output { { set +x; } 2>/dev/null on_err=" echo ERROR: An error was encountered with the build. @@ -15,7 +21,8 @@ exit 1 set -x } -# Copied from ../../shared.sh +# See https://unix.stackexchange.com/questions/82598 +# Duplicated in src/ci/shared.sh function retry { echo "Attempting with retry:" "$@" local n=1 diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 5b9c90273a3..800d8920951 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -470,6 +470,9 @@ jobs: - name: x86_64-gnu-aux <<: *job-linux-4c + - name: x86_64-gnu-integration + <<: *job-linux-16c + - name: x86_64-gnu-debug <<: *job-linux-8c @@ -730,6 +733,7 @@ jobs: CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-16c + master: name: master runs-on: ubuntu-latest diff --git a/src/ci/shared.sh b/src/ci/shared.sh index 8a88c56194c..720394af249 100644 --- a/src/ci/shared.sh +++ b/src/ci/shared.sh @@ -8,7 +8,7 @@ export MIRRORS_BASE="https://ci-mirrors.rust-lang.org/rustc" # See https://unix.stackexchange.com/questions/82598 -# Duplicated in docker/dist-various-2/shared.sh +# Duplicated in docker/scripts/shared.sh function retry { echo "Attempting with retry:" "$@" local n=1