Auto merge of #117313 - GuillaumeGomez:cg_gcc-tests, r=onur-ozkan

Run part of `rustc_codegen_gcc`'s tests in CI

Thanks to #112701 and `@bjorn3,` it made this much easier.

Also cc `@antoyo.`

r? `@bjorn3`

.github/workflows/ci.yml

@@ -57,8 +57,9 @@ jobs:
             os: ubuntu-20.04-4core-16gb
             env: {}
           - name: x86_64-gnu-llvm-15
+            env:
+              ENABLE_GCC_CODEGEN: "1"
             os: ubuntu-20.04-16core-64gb
-            env: {}
           - name: x86_64-gnu-tools
             os: ubuntu-20.04-16core-64gb
             env: {}

compiler/rustc_codegen_gcc/Cargo.toml

@@ -18,7 +18,6 @@ path = "tests/lang_tests_release.rs"
 harness = false
 
 [features]
-default = ["master"]
 master = ["gccjit/master"]
 
 [dependencies]

compiler/rustc_codegen_gcc/config.sh

@@ -25,7 +25,7 @@ else
     exit 1
 fi
 
-HOST_TRIPLE=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ")
+HOST_TRIPLE=$($RUSTC -vV | grep host | cut -d: -f2 | tr -d " ")
 # TODO: remove $OVERWRITE_TARGET_TRIPLE when config.sh is removed.
 TARGET_TRIPLE="${OVERWRITE_TARGET_TRIPLE:-$HOST_TRIPLE}"
 
@@ -54,6 +54,10 @@ if [[ -z "$BUILTIN_BACKEND" ]]; then
     export RUSTFLAGS="$CG_RUSTFLAGS $linker -Csymbol-mangling-version=v0 -Cdebuginfo=2 $disable_lto_flags -Zcodegen-backend=$(pwd)/target/${CHANNEL:-debug}/librustc_codegen_gcc.$dylib_ext --sysroot $(pwd)/build_sysroot/sysroot $TEST_FLAGS"
 else
     export RUSTFLAGS="$CG_RUSTFLAGS $linker -Csymbol-mangling-version=v0 -Cdebuginfo=2 $disable_lto_flags -Zcodegen-backend=gcc $TEST_FLAGS -Cpanic=abort"
+
+    if [[ ! -z "$RUSTC_SYSROOT" ]]; then
+        export RUSTFLAGS="$RUSTFLAGS --sysroot $RUSTC_SYSROOT"
+    fi
 fi
 
 # FIXME(antoyo): remove once the atomic shim is gone

compiler/rustc_codegen_gcc/example/alloc_example.rs

@@ -18,7 +18,7 @@
 }
 
 #[panic_handler]
-fn panic_handler(_: &core::panic::PanicInfo) -> ! {
+fn panic_handler(_: &core::panic::PanicInfo<'_>) -> ! {
     core::intrinsics::abort();
 }
 

compiler/rustc_codegen_gcc/example/alloc_system.rs

@@ -3,7 +3,6 @@
 
 #![no_std]
 #![feature(allocator_api, rustc_private)]
-#![cfg_attr(any(unix, target_os = "redox"), feature(libc))]
 
 // The minimum alignment guaranteed by the architecture. This value is used to
 // add fast paths for low alignment values.
@@ -48,7 +47,18 @@ pub(crate) unsafe fn realloc_fallback(&self, ptr: *mut u8, old_layout: Layout,
 }
 #[cfg(any(unix, target_os = "redox"))]
 mod platform {
-    extern crate libc;
+    mod libc {
+        use core::ffi::{c_void, c_int};
+
+        #[link(name = "c")]
+        extern "C" {
+            pub fn malloc(size: usize) -> *mut c_void;
+            pub fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void;
+            pub fn calloc(nmemb: usize, size: usize) -> *mut c_void;
+            pub fn free(ptr: *mut u8);
+            pub fn posix_memalign(memptr: *mut *mut c_void, alignment: usize, size: usize) -> c_int;
+        }
+    }
     use core::ptr;
     use MIN_ALIGN;
     use System;
@@ -82,12 +92,12 @@ unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
         }
         #[inline]
         unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) {
-            libc::free(ptr as *mut libc::c_void)
+            libc::free(ptr as *mut _)
         }
         #[inline]
         unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
             if layout.align() <= MIN_ALIGN && layout.align() <= new_size {
-                libc::realloc(ptr as *mut libc::c_void, new_size) as *mut u8
+                libc::realloc(ptr as *mut _, new_size) as *mut u8
             } else {
                 self.realloc_fallback(ptr, layout, new_size)
             }

compiler/rustc_codegen_gcc/example/mod_bench.rs

@@ -6,7 +6,7 @@
 extern {}
 
 #[panic_handler]
-fn panic_handler(_: &core::panic::PanicInfo) -> ! {
+fn panic_handler(_: &core::panic::PanicInfo<'_>) -> ! {
     core::intrinsics::abort();
 }
 

src/bootstrap/configure.py

@@ -59,6 +59,7 @@ o("missing-tools", "dist.missing-tools", "allow failures when building tools")
 o("use-libcxx", "llvm.use-libcxx", "build LLVM with libc++")
 o("control-flow-guard", "rust.control-flow-guard", "Enable Control Flow Guard")
 o("patch-binaries-for-nix", "build.patch-binaries-for-nix", "whether patch binaries for usage with Nix toolchains")
+o("new-symbol-mangling", "rust.new-symbol-mangling", "use symbol-mangling-version v0")
 
 v("llvm-cflags", "llvm.cflags", "build LLVM with these extra compiler flags")
 v("llvm-cxxflags", "llvm.cxxflags", "build LLVM with these extra compiler flags")

src/bootstrap/src/core/build_steps/compile.rs

@@ -45,15 +45,42 @@ pub struct Std {
     /// When using download-rustc, we need to use a new build of `std` for running unit tests of Std itself,
     /// but we need to use the downloaded copy of std for linking to rustdoc. Allow this to be overriden by `builder.ensure` from other steps.
     force_recompile: bool,
+    extra_rust_args: &'static [&'static str],
 }
 
 impl Std {
     pub fn new(compiler: Compiler, target: TargetSelection) -> Self {
-        Self { target, compiler, crates: Default::default(), force_recompile: false }
+        Self {
+            target,
+            compiler,
+            crates: Default::default(),
+            force_recompile: false,
+            extra_rust_args: &[],
+        }
     }
 
     pub fn force_recompile(compiler: Compiler, target: TargetSelection) -> Self {
-        Self { target, compiler, crates: Default::default(), force_recompile: true }
+        Self {
+            target,
+            compiler,
+            crates: Default::default(),
+            force_recompile: true,
+            extra_rust_args: &[],
+        }
+    }
+
+    pub fn new_with_extra_rust_args(
+        compiler: Compiler,
+        target: TargetSelection,
+        extra_rust_args: &'static [&'static str],
+    ) -> Self {
+        Self {
+            target,
+            compiler,
+            crates: Default::default(),
+            force_recompile: false,
+            extra_rust_args,
+        }
     }
 }
 
@@ -81,6 +108,7 @@ fn make_run(run: RunConfig<'_>) {
             target: run.target,
             crates,
             force_recompile: false,
+            extra_rust_args: &[],
         });
     }
 
@@ -188,6 +216,9 @@ fn run(self, builder: &Builder<'_>) {
         if target.is_synthetic() {
             cargo.env("RUSTC_BOOTSTRAP_SYNTHETIC_TARGET", "1");
         }
+        for rustflag in self.extra_rust_args.into_iter() {
+            cargo.rustflag(rustflag);
+        }
 
         let _guard = builder.msg(
             Kind::Build,

src/bootstrap/src/core/build_steps/test.rs

@@ -3063,6 +3063,7 @@ fn run(self, builder: &Builder<'_>) {
         // FIXME handle vendoring for source tarballs before removing the --skip-test below
         let download_dir = builder.out.join("cg_clif_download");
 
+        // FIXME: Uncomment the `prepare` command below once vendoring is implemented.
         /*
         let mut prepare_cargo = build_cargo();
         prepare_cargo.arg("--").arg("prepare").arg("--download-dir").arg(&download_dir);
@@ -3094,3 +3095,123 @@ fn run(self, builder: &Builder<'_>) {
         builder.run_cmd(BootstrapCommand::from(&mut cmd).fail_fast());
     }
 }
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct CodegenGCC {
+    compiler: Compiler,
+    target: TargetSelection,
+}
+
+impl Step for CodegenGCC {
+    type Output = ();
+    const DEFAULT: bool = true;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.paths(&["compiler/rustc_codegen_gcc"])
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        let builder = run.builder;
+        let host = run.build_triple();
+        let compiler = run.builder.compiler_for(run.builder.top_stage, host, host);
+
+        if builder.doc_tests == DocTests::Only {
+            return;
+        }
+
+        let triple = run.target.triple;
+        let target_supported =
+            if triple.contains("linux") { triple.contains("x86_64") } else { false };
+        if !target_supported {
+            builder.info("target not supported by rustc_codegen_gcc. skipping");
+            return;
+        }
+
+        if builder.remote_tested(run.target) {
+            builder.info("remote testing is not supported by rustc_codegen_gcc. skipping");
+            return;
+        }
+
+        if !builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("gcc")) {
+            builder.info("gcc not in rust.codegen-backends. skipping");
+            return;
+        }
+
+        builder.ensure(CodegenGCC { compiler, target: run.target });
+    }
+
+    fn run(self, builder: &Builder<'_>) {
+        let compiler = self.compiler;
+        let target = self.target;
+
+        builder.ensure(compile::Std::new_with_extra_rust_args(
+            compiler,
+            target,
+            &["-Csymbol-mangling-version=v0", "-Cpanic=abort"],
+        ));
+
+        // If we're not doing a full bootstrap but we're testing a stage2
+        // version of libstd, then what we're actually testing is the libstd
+        // produced in stage1. Reflect that here by updating the compiler that
+        // we're working with automatically.
+        let compiler = builder.compiler_for(compiler.stage, compiler.host, target);
+
+        let build_cargo = || {
+            let mut cargo = builder.cargo(
+                compiler,
+                Mode::Codegen, // Must be codegen to ensure dlopen on compiled dylibs works
+                SourceType::InTree,
+                target,
+                "run",
+            );
+            cargo.current_dir(&builder.src.join("compiler/rustc_codegen_gcc"));
+            cargo
+                .arg("--manifest-path")
+                .arg(builder.src.join("compiler/rustc_codegen_gcc/build_system/Cargo.toml"));
+            compile::rustc_cargo_env(builder, &mut cargo, target, compiler.stage);
+
+            // Avoid incremental cache issues when changing rustc
+            cargo.env("CARGO_BUILD_INCREMENTAL", "false");
+            cargo.rustflag("-Cpanic=abort");
+
+            cargo
+        };
+
+        builder.info(&format!(
+            "{} GCC stage{} ({} -> {})",
+            Kind::Test.description(),
+            compiler.stage,
+            &compiler.host,
+            target
+        ));
+        let _time = helpers::timeit(&builder);
+
+        // FIXME: Uncomment the `prepare` command below once vendoring is implemented.
+        /*
+        let mut prepare_cargo = build_cargo();
+        prepare_cargo.arg("--").arg("prepare");
+        #[allow(deprecated)]
+        builder.config.try_run(&mut prepare_cargo.into()).unwrap();
+        */
+
+        let mut cargo = build_cargo();
+
+        cargo
+            .arg("--")
+            .arg("test")
+            .arg("--use-system-gcc")
+            .arg("--use-backend")
+            .arg("gcc")
+            .arg("--out-dir")
+            .arg(builder.stage_out(compiler, Mode::ToolRustc).join("cg_gcc"))
+            .arg("--release")
+            .arg("--no-default-features")
+            .arg("--mini-tests")
+            .arg("--std-tests");
+        cargo.args(builder.config.test_args());
+
+        let mut cmd: Command = cargo.into();
+        builder.run_cmd(BootstrapCommand::from(&mut cmd).fail_fast());
+    }
+}

src/bootstrap/src/core/builder.rs

@@ -738,6 +738,7 @@ macro_rules! describe {
                 test::Debuginfo,
                 test::UiFullDeps,
                 test::CodegenCranelift,
+                test::CodegenGCC,
                 test::Rustdoc,
                 test::RunCoverageRustdoc,
                 test::Pretty,

src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile

@@ -24,6 +24,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   xz-utils \
   nodejs \
   mingw-w64 \
+  libgccjit-12-dev \
   && rm -rf /var/lib/apt/lists/*
 
 # Install powershell (universal package) so we can test x.ps1 on Linux
@@ -34,6 +35,9 @@ RUN curl -sL "https://github.com/PowerShell/PowerShell/releases/download/v7.3.1/
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
+# Make `libgccjit.so` accessible to the linker.
+RUN ln -s /usr/lib/gcc/x86_64-linux-gnu/12/libgccjit.so /usr/lib/x86_64-linux-gnu/libgccjit.so
+
 # We are disabling CI LLVM since this builder is intentionally using a host
 # LLVM, rather than the typical src/llvm-project LLVM.
 ENV NO_DOWNLOAD_CI_LLVM 1
@@ -47,6 +51,7 @@ ENV RUST_CONFIGURE_ARGS \
       --build=x86_64-unknown-linux-gnu \
       --llvm-root=/usr/lib/llvm-15 \
       --enable-llvm-link-shared \
+      $USE_NEW_MANGLING \
       --set rust.thin-lto-import-instr-limit=10
 
 COPY host-x86_64/x86_64-gnu-llvm-15/script.sh /tmp/

src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh

@@ -4,7 +4,13 @@ set -ex
 
 # Only run the stage 1 tests on merges, not on PR CI jobs.
 if [[ -z "${PR_CI_JOB}" ]]; then
-    ../x.py --stage 1 test --skip src/tools/tidy
+    # When running gcc backend tests, we need to install `libgccjit` and to not run llvm codegen
+    # tests as it will fail them.
+    if [[ "${ENABLE_GCC_CODEGEN}" == "1" ]]; then
+        ../x.py --stage 1 test --skip src/tools/tidy --skip tests/codegen
+    else
+        ../x.py --stage 1 test --skip src/tools/tidy
+    fi
 
     # Run the `mir-opt` tests again but this time for a 32-bit target.
     # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have
@@ -19,8 +25,14 @@ if [[ -z "${PR_CI_JOB}" ]]; then
     ../x.py --stage 1 test tests/ui-fulldeps
 fi
 
+# When running gcc backend tests, we need to install `libgccjit` and to not run llvm codegen
+# tests as it will fail them.
 # NOTE: intentionally uses all of `x.py`, `x`, and `x.ps1` to make sure they all work on Linux.
-../x.py --stage 2 test --skip src/tools/tidy
+if [[ "${ENABLE_GCC_CODEGEN}" == "1" ]]; then
+    ../x.py --stage 2 test --skip src/tools/tidy --skip tests/codegen
+else
+    ../x.py --stage 2 test --skip src/tools/tidy
+fi
 
 # Run the `mir-opt` tests again but this time for a 32-bit target.
 # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have

src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile

@@ -15,6 +15,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   sudo \
   xz-utils \
   tidy \
+  libgccjit-12-dev \
   \
 # Install dependencies for chromium browser
   gconf-service \
@@ -61,6 +62,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
+# Make `libgccjit.so` accessible.
+RUN ln -s /usr/lib/gcc/x86_64-linux-gnu/12/libgccjit.so /usr/lib/x86_64-linux-gnu/libgccjit.so
+# Fix rustc_codegen_gcc lto issues.
+ENV GCC_EXEC_PREFIX="/usr/lib/gcc/"
+
 COPY host-x86_64/x86_64-gnu-tools/checktools.sh /tmp/
 
 RUN curl -sL https://nodejs.org/dist/v14.20.0/node-v14.20.0-linux-x64.tar.xz | tar -xJ
@@ -81,7 +87,8 @@ RUN npm install -g browser-ui-test@$(head -n 1 /tmp/browser-ui-test.version) --u
 
 ENV RUST_CONFIGURE_ARGS \
   --build=x86_64-unknown-linux-gnu \
-  --save-toolstates=/tmp/toolstate/toolstates.json
+  --save-toolstates=/tmp/toolstate/toolstates.json \
+  --enable-new-symbol-mangling
 
 ENV HOST_TARGET x86_64-unknown-linux-gnu
 

src/ci/docker/run.sh

@@ -267,10 +267,24 @@ fi
 SUMMARY_FILE=github-summary.md
 touch $objdir/${SUMMARY_FILE}
 
+extra_env=""
+if [ "$ENABLE_GCC_CODEGEN" = "1" ]; then
+  extra_env="$EXTRA_ENV --env ENABLE_GCC_CODEGEN=1"
+  # If `ENABLE_GCC_CODEGEN` is set and not empty, we add the `--enable-new-symbol-mangling`
+  # argument to `RUST_CONFIGURE_ARGS` and set the `GCC_EXEC_PREFIX` environment variable.
+  # `cg_gcc` doesn't support the legacy mangling so we need to enforce the new one
+  # if we run `cg_gcc` tests.
+  extra_env="$EXTRA_ENV --env USE_NEW_MANGLING=--enable-new-symbol-mangling"
+  # Fix rustc_codegen_gcc lto issues.
+  extra_env="$EXTRA_ENV --env GCC_EXEC_PREFIX=/usr/lib/gcc/"
+  echo "Setting extra environment values for docker: $extra_env"
+fi
+
 docker \
   run \
   --workdir /checkout/obj \
   --env SRC=/checkout \
+  $extra_env \
   $args \
   --env CARGO_HOME=/cargo \
   --env DEPLOY \

src/ci/github-actions/ci.yml

@@ -331,6 +331,8 @@ jobs:
             <<: *job-linux-4c
 
           - name: x86_64-gnu-llvm-15
+            env:
+              ENABLE_GCC_CODEGEN: "1"
             <<: *job-linux-16c
 
           - name: x86_64-gnu-tools

src/ci/run.sh

@@ -126,8 +126,15 @@ else
 
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir"
 
-  # Test the Cranelift backend in CI. Bootstrap knows which targets to run tests on.
+  # When running gcc backend tests, we need to install `libgccjit` and to not run llvm codegen
-  RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift"
+  # tests as it will fail them.
+  if [[ "${ENABLE_GCC_CODEGEN}" == "1" ]]; then
+    # Test the Cranelift and GCC backends in CI. Bootstrap knows which targets to run tests on.
+    RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift,gcc"
+  else
+    # Test the Cranelift backend in CI. Bootstrap knows which targets to run tests on.
+    RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift"
+  fi
 
   # We enable this for non-dist builders, since those aren't trying to produce
   # fresh binaries. We currently don't entirely support distributing a fresh