Auto merge of #117313 - GuillaumeGomez:cg_gcc-tests, r=onur-ozkan

Run part of `rustc_codegen_gcc`'s tests in CI Thanks to #112701 and `@bjorn3,` it made this much easier. Also cc `@antoyo.` r? `@bjorn3`
2023-11-03 02:05:06 +00:00 · 2023-11-03 02:05:06 +00:00 · 6b9d6dedd0
commit 6b9d6dedd0
parent 2520ca8566 c890dd66b3
16 changed files with 231 additions and 16 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -57,8 +57,9 @@ jobs:
            os: ubuntu-20.04-4core-16gb
            env: {}
          - name: x86_64-gnu-llvm-15
+            env:
+              ENABLE_GCC_CODEGEN: "1"
            os: ubuntu-20.04-16core-64gb
-            env: {}
          - name: x86_64-gnu-tools
            os: ubuntu-20.04-16core-64gb
            env: {}
--- a/compiler/rustc_codegen_gcc/Cargo.toml
+++ b/compiler/rustc_codegen_gcc/Cargo.toml
@ -18,7 +18,6 @@ path = "tests/lang_tests_release.rs"
 harness = false

 [features]
-default = ["master"]
 master = ["gccjit/master"]

 [dependencies]
--- a/compiler/rustc_codegen_gcc/config.sh
+++ b/compiler/rustc_codegen_gcc/config.sh
@ -25,7 +25,7 @@ else
    exit 1
 fi

-HOST_TRIPLE=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ")
+HOST_TRIPLE=$($RUSTC -vV | grep host | cut -d: -f2 | tr -d " ")
 # TODO: remove $OVERWRITE_TARGET_TRIPLE when config.sh is removed.
 TARGET_TRIPLE="${OVERWRITE_TARGET_TRIPLE:-$HOST_TRIPLE}"

@ -54,6 +54,10 @@ if [[ -z "$BUILTIN_BACKEND" ]]; then
    export RUSTFLAGS="$CG_RUSTFLAGS $linker -Csymbol-mangling-version=v0 -Cdebuginfo=2 $disable_lto_flags -Zcodegen-backend=$(pwd)/target/${CHANNEL:-debug}/librustc_codegen_gcc.$dylib_ext --sysroot $(pwd)/build_sysroot/sysroot $TEST_FLAGS"
 else
    export RUSTFLAGS="$CG_RUSTFLAGS $linker -Csymbol-mangling-version=v0 -Cdebuginfo=2 $disable_lto_flags -Zcodegen-backend=gcc $TEST_FLAGS -Cpanic=abort"
+
+    if [[ ! -z "$RUSTC_SYSROOT" ]]; then
+        export RUSTFLAGS="$RUSTFLAGS --sysroot $RUSTC_SYSROOT"
+    fi
 fi

 # FIXME(antoyo): remove once the atomic shim is gone
--- a/compiler/rustc_codegen_gcc/example/alloc_example.rs
+++ b/compiler/rustc_codegen_gcc/example/alloc_example.rs
@ -18,7 +18,7 @@
 }

 #[panic_handler]
-fn panic_handler(_: &core::panic::PanicInfo) -> ! {
+fn panic_handler(_: &core::panic::PanicInfo<'_>) -> ! {
    core::intrinsics::abort();
 }

--- a/compiler/rustc_codegen_gcc/example/alloc_system.rs
+++ b/compiler/rustc_codegen_gcc/example/alloc_system.rs
@ -3,7 +3,6 @@

 #![no_std]
 #![feature(allocator_api, rustc_private)]
-#![cfg_attr(any(unix, target_os = "redox"), feature(libc))]

 // The minimum alignment guaranteed by the architecture. This value is used to
 // add fast paths for low alignment values.
@ -48,7 +47,18 @@ pub(crate) unsafe fn realloc_fallback(&self, ptr: *mut u8, old_layout: Layout,
 }
 #[cfg(any(unix, target_os = "redox"))]
 mod platform {
-    extern crate libc;
+    mod libc {
+        use core::ffi::{c_void, c_int};
+
+        #[link(name = "c")]
+        extern "C" {
+            pub fn malloc(size: usize) -> *mut c_void;
+            pub fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void;
+            pub fn calloc(nmemb: usize, size: usize) -> *mut c_void;
+            pub fn free(ptr: *mut u8);
+            pub fn posix_memalign(memptr: *mut *mut c_void, alignment: usize, size: usize) -> c_int;
+        }
+    }
    use core::ptr;
    use MIN_ALIGN;
    use System;
@ -82,12 +92,12 @@ unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
        }
        #[inline]
        unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) {
-            libc::free(ptr as *mut libc::c_void)
+            libc::free(ptr as *mut _)
        }
        #[inline]
        unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
            if layout.align() <= MIN_ALIGN && layout.align() <= new_size {
-                libc::realloc(ptr as *mut libc::c_void, new_size) as *mut u8
+                libc::realloc(ptr as *mut _, new_size) as *mut u8
            } else {
                self.realloc_fallback(ptr, layout, new_size)
            }
--- a/compiler/rustc_codegen_gcc/example/mod_bench.rs
+++ b/compiler/rustc_codegen_gcc/example/mod_bench.rs
@ -6,7 +6,7 @@
 extern {}

 #[panic_handler]
-fn panic_handler(_: &core::panic::PanicInfo) -> ! {
+fn panic_handler(_: &core::panic::PanicInfo<'_>) -> ! {
    core::intrinsics::abort();
 }

--- a/src/bootstrap/configure.py
+++ b/src/bootstrap/configure.py
@ -59,6 +59,7 @@ o("missing-tools", "dist.missing-tools", "allow failures when building tools")
 o("use-libcxx", "llvm.use-libcxx", "build LLVM with libc++")
 o("control-flow-guard", "rust.control-flow-guard", "Enable Control Flow Guard")
 o("patch-binaries-for-nix", "build.patch-binaries-for-nix", "whether patch binaries for usage with Nix toolchains")
+o("new-symbol-mangling", "rust.new-symbol-mangling", "use symbol-mangling-version v0")

 v("llvm-cflags", "llvm.cflags", "build LLVM with these extra compiler flags")
 v("llvm-cxxflags", "llvm.cxxflags", "build LLVM with these extra compiler flags")
--- a/src/bootstrap/src/core/build_steps/compile.rs
+++ b/src/bootstrap/src/core/build_steps/compile.rs
@ -45,15 +45,42 @@ pub struct Std {
    /// When using download-rustc, we need to use a new build of `std` for running unit tests of Std itself,
    /// but we need to use the downloaded copy of std for linking to rustdoc. Allow this to be overriden by `builder.ensure` from other steps.
    force_recompile: bool,
+    extra_rust_args: &'static [&'static str],
 }

 impl Std {
    pub fn new(compiler: Compiler, target: TargetSelection) -> Self {
-        Self { target, compiler, crates: Default::default(), force_recompile: false }
+        Self {
+            target,
+            compiler,
+            crates: Default::default(),
+            force_recompile: false,
+            extra_rust_args: &[],
+        }
    }

    pub fn force_recompile(compiler: Compiler, target: TargetSelection) -> Self {
-        Self { target, compiler, crates: Default::default(), force_recompile: true }
+        Self {
+            target,
+            compiler,
+            crates: Default::default(),
+            force_recompile: true,
+            extra_rust_args: &[],
+        }
+    }
+
+    pub fn new_with_extra_rust_args(
+        compiler: Compiler,
+        target: TargetSelection,
+        extra_rust_args: &'static [&'static str],
+    ) -> Self {
+        Self {
+            target,
+            compiler,
+            crates: Default::default(),
+            force_recompile: false,
+            extra_rust_args,
+        }
    }
 }

@ -81,6 +108,7 @@ fn make_run(run: RunConfig<'_>) {
            target: run.target,
            crates,
            force_recompile: false,
+            extra_rust_args: &[],
        });
    }

@ -188,6 +216,9 @@ fn run(self, builder: &Builder<'_>) {
        if target.is_synthetic() {
            cargo.env("RUSTC_BOOTSTRAP_SYNTHETIC_TARGET", "1");
        }
+        for rustflag in self.extra_rust_args.into_iter() {
+            cargo.rustflag(rustflag);
+        }

        let _guard = builder.msg(
            Kind::Build,
--- a/src/bootstrap/src/core/build_steps/test.rs
+++ b/src/bootstrap/src/core/build_steps/test.rs
@ -3063,6 +3063,7 @@ fn run(self, builder: &Builder<'_>) {
        // FIXME handle vendoring for source tarballs before removing the --skip-test below
        let download_dir = builder.out.join("cg_clif_download");

+        // FIXME: Uncomment the `prepare` command below once vendoring is implemented.
        /*
        let mut prepare_cargo = build_cargo();
        prepare_cargo.arg("--").arg("prepare").arg("--download-dir").arg(&download_dir);
@ -3094,3 +3095,123 @@ fn run(self, builder: &Builder<'_>) {
        builder.run_cmd(BootstrapCommand::from(&mut cmd).fail_fast());
    }
 }
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct CodegenGCC {
+    compiler: Compiler,
+    target: TargetSelection,
+}
+
+impl Step for CodegenGCC {
+    type Output = ();
+    const DEFAULT: bool = true;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.paths(&["compiler/rustc_codegen_gcc"])
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        let builder = run.builder;
+        let host = run.build_triple();
+        let compiler = run.builder.compiler_for(run.builder.top_stage, host, host);
+
+        if builder.doc_tests == DocTests::Only {
+            return;
+        }
+
+        let triple = run.target.triple;
+        let target_supported =
+            if triple.contains("linux") { triple.contains("x86_64") } else { false };
+        if !target_supported {
+            builder.info("target not supported by rustc_codegen_gcc. skipping");
+            return;
+        }
+
+        if builder.remote_tested(run.target) {
+            builder.info("remote testing is not supported by rustc_codegen_gcc. skipping");
+            return;
+        }
+
+        if !builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("gcc")) {
+            builder.info("gcc not in rust.codegen-backends. skipping");
+            return;
+        }
+
+        builder.ensure(CodegenGCC { compiler, target: run.target });
+    }
+
+    fn run(self, builder: &Builder<'_>) {
+        let compiler = self.compiler;
+        let target = self.target;
+
+        builder.ensure(compile::Std::new_with_extra_rust_args(
+            compiler,
+            target,
+            &["-Csymbol-mangling-version=v0", "-Cpanic=abort"],
+        ));
+
+        // If we're not doing a full bootstrap but we're testing a stage2
+        // version of libstd, then what we're actually testing is the libstd
+        // produced in stage1. Reflect that here by updating the compiler that
+        // we're working with automatically.
+        let compiler = builder.compiler_for(compiler.stage, compiler.host, target);
+
+        let build_cargo = || {
+            let mut cargo = builder.cargo(
+                compiler,
+                Mode::Codegen, // Must be codegen to ensure dlopen on compiled dylibs works
+                SourceType::InTree,
+                target,
+                "run",
+            );
+            cargo.current_dir(&builder.src.join("compiler/rustc_codegen_gcc"));
+            cargo
+                .arg("--manifest-path")
+                .arg(builder.src.join("compiler/rustc_codegen_gcc/build_system/Cargo.toml"));
+            compile::rustc_cargo_env(builder, &mut cargo, target, compiler.stage);
+
+            // Avoid incremental cache issues when changing rustc
+            cargo.env("CARGO_BUILD_INCREMENTAL", "false");
+            cargo.rustflag("-Cpanic=abort");
+
+            cargo
+        };
+
+        builder.info(&format!(
+            "{} GCC stage{} ({} -> {})",
+            Kind::Test.description(),
+            compiler.stage,
+            &compiler.host,
+            target
+        ));
+        let _time = helpers::timeit(&builder);
+
+        // FIXME: Uncomment the `prepare` command below once vendoring is implemented.
+        /*
+        let mut prepare_cargo = build_cargo();
+        prepare_cargo.arg("--").arg("prepare");
+        #[allow(deprecated)]
+        builder.config.try_run(&mut prepare_cargo.into()).unwrap();
+        */
+
+        let mut cargo = build_cargo();
+
+        cargo
+            .arg("--")
+            .arg("test")
+            .arg("--use-system-gcc")
+            .arg("--use-backend")
+            .arg("gcc")
+            .arg("--out-dir")
+            .arg(builder.stage_out(compiler, Mode::ToolRustc).join("cg_gcc"))
+            .arg("--release")
+            .arg("--no-default-features")
+            .arg("--mini-tests")
+            .arg("--std-tests");
+        cargo.args(builder.config.test_args());
+
+        let mut cmd: Command = cargo.into();
+        builder.run_cmd(BootstrapCommand::from(&mut cmd).fail_fast());
+    }
+}
--- a/src/bootstrap/src/core/builder.rs
+++ b/src/bootstrap/src/core/builder.rs
@ -738,6 +738,7 @@ macro_rules! describe {
                test::Debuginfo,
                test::UiFullDeps,
                test::CodegenCranelift,
+                test::CodegenGCC,
                test::Rustdoc,
                test::RunCoverageRustdoc,
                test::Pretty,
--- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile
@ -24,6 +24,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
  xz-utils \
  nodejs \
  mingw-w64 \
+  libgccjit-12-dev \
  && rm -rf /var/lib/apt/lists/*

 # Install powershell (universal package) so we can test x.ps1 on Linux
@ -34,6 +35,9 @@ RUN curl -sL "https://github.com/PowerShell/PowerShell/releases/download/v7.3.1/
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh

+# Make `libgccjit.so` accessible to the linker.
+RUN ln -s /usr/lib/gcc/x86_64-linux-gnu/12/libgccjit.so /usr/lib/x86_64-linux-gnu/libgccjit.so
+
 # We are disabling CI LLVM since this builder is intentionally using a host
 # LLVM, rather than the typical src/llvm-project LLVM.
 ENV NO_DOWNLOAD_CI_LLVM 1
@ -47,6 +51,7 @@ ENV RUST_CONFIGURE_ARGS \
      --build=x86_64-unknown-linux-gnu \
      --llvm-root=/usr/lib/llvm-15 \
      --enable-llvm-link-shared \
+      $USE_NEW_MANGLING \
      --set rust.thin-lto-import-instr-limit=10

 COPY host-x86_64/x86_64-gnu-llvm-15/script.sh /tmp/
--- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh
@ -4,7 +4,13 @@ set -ex

 # Only run the stage 1 tests on merges, not on PR CI jobs.
 if [[ -z "${PR_CI_JOB}" ]]; then
-    ../x.py --stage 1 test --skip src/tools/tidy
+    # When running gcc backend tests, we need to install `libgccjit` and to not run llvm codegen
+    # tests as it will fail them.
+    if [[ "${ENABLE_GCC_CODEGEN}" == "1" ]]; then
+        ../x.py --stage 1 test --skip src/tools/tidy --skip tests/codegen
+    else
+        ../x.py --stage 1 test --skip src/tools/tidy
+    fi

    # Run the `mir-opt` tests again but this time for a 32-bit target.
    # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have
@ -19,8 +25,14 @@ if [[ -z "${PR_CI_JOB}" ]]; then
    ../x.py --stage 1 test tests/ui-fulldeps
 fi

+# When running gcc backend tests, we need to install `libgccjit` and to not run llvm codegen
+# tests as it will fail them.
 # NOTE: intentionally uses all of `x.py`, `x`, and `x.ps1` to make sure they all work on Linux.
-../x.py --stage 2 test --skip src/tools/tidy
+if [[ "${ENABLE_GCC_CODEGEN}" == "1" ]]; then
+    ../x.py --stage 2 test --skip src/tools/tidy --skip tests/codegen
+else
+    ../x.py --stage 2 test --skip src/tools/tidy
+fi

 # Run the `mir-opt` tests again but this time for a 32-bit target.
 # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have
--- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile
@ -15,6 +15,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
  sudo \
  xz-utils \
  tidy \
+  libgccjit-12-dev \
  \
 # Install dependencies for chromium browser
  gconf-service \
@ -61,6 +62,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh

+# Make `libgccjit.so` accessible.
+RUN ln -s /usr/lib/gcc/x86_64-linux-gnu/12/libgccjit.so /usr/lib/x86_64-linux-gnu/libgccjit.so
+# Fix rustc_codegen_gcc lto issues.
+ENV GCC_EXEC_PREFIX="/usr/lib/gcc/"
+
 COPY host-x86_64/x86_64-gnu-tools/checktools.sh /tmp/

 RUN curl -sL https://nodejs.org/dist/v14.20.0/node-v14.20.0-linux-x64.tar.xz | tar -xJ
@ -81,7 +87,8 @@ RUN npm install -g browser-ui-test@$(head -n 1 /tmp/browser-ui-test.version) --u

 ENV RUST_CONFIGURE_ARGS \
  --build=x86_64-unknown-linux-gnu \
-  --save-toolstates=/tmp/toolstate/toolstates.json
+  --save-toolstates=/tmp/toolstate/toolstates.json \
+  --enable-new-symbol-mangling

 ENV HOST_TARGET x86_64-unknown-linux-gnu

--- a/src/ci/docker/run.sh
+++ b/src/ci/docker/run.sh
@ -267,10 +267,24 @@ fi
 SUMMARY_FILE=github-summary.md
 touch $objdir/${SUMMARY_FILE}

+extra_env=""
+if [ "$ENABLE_GCC_CODEGEN" = "1" ]; then
+  extra_env="$EXTRA_ENV --env ENABLE_GCC_CODEGEN=1"
+  # If `ENABLE_GCC_CODEGEN` is set and not empty, we add the `--enable-new-symbol-mangling`
+  # argument to `RUST_CONFIGURE_ARGS` and set the `GCC_EXEC_PREFIX` environment variable.
+  # `cg_gcc` doesn't support the legacy mangling so we need to enforce the new one
+  # if we run `cg_gcc` tests.
+  extra_env="$EXTRA_ENV --env USE_NEW_MANGLING=--enable-new-symbol-mangling"
+  # Fix rustc_codegen_gcc lto issues.
+  extra_env="$EXTRA_ENV --env GCC_EXEC_PREFIX=/usr/lib/gcc/"
+  echo "Setting extra environment values for docker: $extra_env"
+fi
+
 docker \
  run \
  --workdir /checkout/obj \
  --env SRC=/checkout \
+  $extra_env \
  $args \
  --env CARGO_HOME=/cargo \
  --env DEPLOY \
--- a/src/ci/github-actions/ci.yml
+++ b/src/ci/github-actions/ci.yml
@ -331,6 +331,8 @@ jobs:
            <<: *job-linux-4c

          - name: x86_64-gnu-llvm-15
+            env:
+              ENABLE_GCC_CODEGEN: "1"
            <<: *job-linux-16c

          - name: x86_64-gnu-tools
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@ -126,8 +126,15 @@ else

  RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir"

-  # Test the Cranelift backend in CI. Bootstrap knows which targets to run tests on.
-  RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift"
+  # When running gcc backend tests, we need to install `libgccjit` and to not run llvm codegen
+  # tests as it will fail them.
+  if [[ "${ENABLE_GCC_CODEGEN}" == "1" ]]; then
+    # Test the Cranelift and GCC backends in CI. Bootstrap knows which targets to run tests on.
+    RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift,gcc"
+  else
+    # Test the Cranelift backend in CI. Bootstrap knows which targets to run tests on.
+    RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift"
+  fi

  # We enable this for non-dist builders, since those aren't trying to produce
  # fresh binaries. We currently don't entirely support distributing a fresh