From 5adaed06a0326a567ce0571ba1cdbf503f99bf86 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 30 Jun 2024 11:28:14 +0000 Subject: [PATCH] Merge commit '49cd5dd454d0115cfbe9e39102a8b3ba4616aa40' into sync_cg_clif-2024-06-30 --- .github/workflows/abi-cafe.yml | 4 - .github/workflows/main.yml | 17 ++-- Cargo.lock | 58 ++++++------- Cargo.toml | 14 ++-- Readme.md | 3 +- build_system/build_sysroot.rs | 35 -------- build_system/tests.rs | 36 +------- example/mini_core_hello_world.rs | 20 ++--- example/std_example.rs | 38 +++++++++ patches/stdlib-lock.toml | 33 +++++--- rust-toolchain | 2 +- scripts/test_rustc_tests.sh | 30 ++++--- src/abi/mod.rs | 137 +++++++++++++++++++++++-------- src/allocator.rs | 12 +-- src/base.rs | 5 +- src/common.rs | 4 +- src/driver/aot.rs | 31 +++---- src/driver/jit.rs | 52 ++++-------- src/inline_asm.rs | 16 +--- src/intrinsics/llvm_x86.rs | 41 ++++++++- src/lib.rs | 20 ++--- src/main_shim.rs | 10 +-- src/unwind_module.rs | 115 ++++++++++++++++++++++++++ 23 files changed, 441 insertions(+), 292 deletions(-) create mode 100644 src/unwind_module.rs diff --git a/.github/workflows/abi-cafe.yml b/.github/workflows/abi-cafe.yml index b7063f35a3e..1ed6f8fc359 100644 --- a/.github/workflows/abi-cafe.yml +++ b/.github/workflows/abi-cafe.yml @@ -55,10 +55,6 @@ jobs: if: matrix.os == 'macos-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-apple-darwin' run: rustup set default-host x86_64-apple-darwin - - name: Select XCode version - if: matrix.os == 'macos-latest' - run: sudo xcode-select -s /Applications/Xcode_14.3.1.app - - name: Prepare dependencies run: ./y.sh prepare diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1f5a6513f63..a2ae3d63fb9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -66,6 +66,9 @@ jobs: env: TARGET_TRIPLE: aarch64-unknown-linux-gnu apt_deps: gcc-aarch64-linux-gnu qemu-user + - os: macos-latest + env: + TARGET_TRIPLE: aarch64-apple-darwin - os: ubuntu-latest env: TARGET_TRIPLE: s390x-unknown-linux-gnu @@ -108,10 +111,6 @@ jobs: sudo apt-get update sudo apt-get install -y ${{ matrix.apt_deps }} - - name: Select XCode version - if: matrix.os == 'macos-latest' - run: sudo xcode-select -s /Applications/Xcode_14.3.1.app - - name: Prepare dependencies run: ./y.sh prepare @@ -214,6 +213,9 @@ jobs: - os: macos-latest env: TARGET_TRIPLE: x86_64-apple-darwin + - os: macos-latest + env: + TARGET_TRIPLE: aarch64-apple-darwin # cross-compile from Linux to Windows using mingw - os: ubuntu-latest env: @@ -248,10 +250,6 @@ jobs: sudo apt-get update sudo apt-get install -y gcc-mingw-w64-x86-64 - - name: Select XCode version - if: matrix.os == 'macos-latest' - run: sudo xcode-select -s /Applications/Xcode_14.3.1.app - - name: Prepare dependencies run: ./y.sh prepare @@ -282,7 +280,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 if: ${{ github.ref == 'refs/heads/master' }} - needs: [rustfmt, test, bench, dist] + # FIXME add the bench job back to the dependency list once rust-lang/rust#125493 gets merged + needs: [rustfmt, test, dist] permissions: contents: write # for creating the dev tag and release diff --git a/Cargo.lock b/Cargo.lock index 33fe52ddbdd..15c9e9d66fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -46,18 +46,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cranelift-bforest" -version = "0.107.0" +version = "0.109.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79b27922a6879b5b5361d0a084cb0b1941bf109a98540addcb932da13b68bed4" +checksum = "0b6b33d7e757a887989eb18b35712b2a67d96171ec3149d1bfb657b29b7b367c" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-codegen" -version = "0.107.0" +version = "0.109.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "304c455b28bf56372729acb356afbb55d622f2b0f2f7837aa5e57c138acaac4d" +checksum = "b9acf15cb22be42d07c3b57d7856329cb228b7315d385346149df2566ad5e4aa" dependencies = [ "bumpalo", "cranelift-bforest", @@ -70,45 +70,46 @@ dependencies = [ "hashbrown 0.14.3", "log", "regalloc2", + "rustc-hash", "smallvec", "target-lexicon", ] [[package]] name = "cranelift-codegen-meta" -version = "0.107.0" +version = "0.109.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1653c56b99591d07f67c5ca7f9f25888948af3f4b97186bff838d687d666f613" +checksum = "e934d301392b73b3f8b0540391fb82465a0f179a3cee7c726482ac4727efcc97" dependencies = [ "cranelift-codegen-shared", ] [[package]] name = "cranelift-codegen-shared" -version = "0.107.0" +version = "0.109.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5b6a9cf6b6eb820ee3f973a0db313c05dc12d370f37b4fe9630286e1672573f" +checksum = "8afb2a2566b3d54b854dfb288b3b187f6d3d17d6f762c92898207eba302931da" [[package]] name = "cranelift-control" -version = "0.107.0" +version = "0.109.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9d06e6bf30075fb6bed9e034ec046475093392eea1aff90eb5c44c4a033d19a" +checksum = "0100f33b704cdacd01ad66ff41f8c5030d57cbff078e2a4e49ab1822591299fa" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.107.0" +version = "0.109.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29be04f931b73cdb9694874a295027471817f26f26d2f0ebe5454153176b6e3a" +checksum = "a8cfdc315e5d18997093e040a8d234bea1ac1e118a716d3e30f40d449e78207b" [[package]] name = "cranelift-frontend" -version = "0.107.0" +version = "0.109.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a07fd7393041d7faa2f37426f5dc7fc04003b70988810e8c063beefeff1cd8f9" +checksum = "0f74b84f16af2e982b0c0c72233503d9d55cbfe3865dbe807ca28dc6642a28b5" dependencies = [ "cranelift-codegen", "log", @@ -118,15 +119,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.107.0" +version = "0.109.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f341d7938caa6dff8149dac05bb2b53fc680323826b83b4cf175ab9f5139a3c9" +checksum = "adf306d3dde705fb94bd48082f01d38c4ededc74293a4c007805f610bf08bc6e" [[package]] name = "cranelift-jit" -version = "0.107.0" +version = "0.109.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42733555e06433f1461570e09dbd756dafc228b4dac75c597cdbdc518de07522" +checksum = "f5c5cfb8bbd3339cd25cca30e7516ff8fe5cb1feeddde6980cc4d5ef34df97bb" dependencies = [ "anyhow", "cranelift-codegen", @@ -144,9 +145,9 @@ dependencies = [ [[package]] name = "cranelift-module" -version = "0.107.0" +version = "0.109.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84950af02bb85f3da764d53a953b43bb29a732e793d4fe24637a61591be9a024" +checksum = "7c9b0d4269b36fd858e6d8f20cd4938941186fb831488c361888cb2d6b33a9a6" dependencies = [ "anyhow", "cranelift-codegen", @@ -155,9 +156,9 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.107.0" +version = "0.109.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82af6066e6448d26eeabb7aa26a43f7ff79f8217b06bade4ee6ef230aecc8880" +checksum = "1ea0ebdef7aff4a79bcbc8b6495f31315f16b3bf311152f472eaa8d679352581" dependencies = [ "cranelift-codegen", "libc", @@ -166,9 +167,9 @@ dependencies = [ [[package]] name = "cranelift-object" -version = "0.107.0" +version = "0.109.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00af56107039ed150391df6f753298c7b08f2b6a2e0727d216b5fa599d684d8b" +checksum = "19e33439ec20db058bc7cc3410f9748ab1ad90a35cef713d625c736f43e3820d" dependencies = [ "anyhow", "cranelift-codegen", @@ -278,9 +279,9 @@ checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "object" -version = "0.33.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8dd6c0cdf9429bce006e1362bfce61fa1bfd8c898a643ed8d2b471934701d3d" +checksum = "081b846d1d56ddfc18fdf1a922e4f6e07a11768ea1b92dec44e42b72712ccfce" dependencies = [ "crc32fast", "hashbrown 0.14.3", @@ -410,10 +411,11 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "wasmtime-jit-icache-coherence" -version = "20.0.0" +version = "22.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a9f93a3289057b26dc75eb84d6e60d7694f7d169c7c09597495de6e016a13ff" +checksum = "5afe2f0499542f9a4bcfa1b55bfdda803b6ade4e7c93c6b99e0f39dba44b0a91" dependencies = [ + "anyhow", "cfg-if", "libc", "windows-sys", diff --git a/Cargo.toml b/Cargo.toml index 2015cdbcc2a..2969a6cf6ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,15 +8,15 @@ crate-type = ["dylib"] [dependencies] # These have to be in sync with each other -cranelift-codegen = { version = "0.107.0", default-features = false, features = ["std", "unwind", "all-arch"] } -cranelift-frontend = { version = "0.107.0" } -cranelift-module = { version = "0.107.0" } -cranelift-native = { version = "0.107.0" } -cranelift-jit = { version = "0.107.0", optional = true } -cranelift-object = { version = "0.107.0" } +cranelift-codegen = { version = "0.109.0", default-features = false, features = ["std", "unwind", "all-arch"] } +cranelift-frontend = { version = "0.109.0" } +cranelift-module = { version = "0.109.0" } +cranelift-native = { version = "0.109.0" } +cranelift-jit = { version = "0.109.0", optional = true } +cranelift-object = { version = "0.109.0" } target-lexicon = "0.12.0" gimli = { version = "0.28", default-features = false, features = ["write"]} -object = { version = "0.33", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] } +object = { version = "0.36", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] } indexmap = "2.0.0" libloading = { version = "0.8.0", optional = true } diff --git a/Readme.md b/Readme.md index 00ea15cb38c..eb21e027dd0 100644 --- a/Readme.md +++ b/Readme.md @@ -70,7 +70,7 @@ For more docs on how to build and test see [build_system/usage.txt](build_system |FreeBSD|✅[^no-rustup]|❓|❓|❓| |AIX|❌[^xcoff]|N/A|N/A|❌[^xcoff]| |Other unixes|❓|❓|❓|❓| -|macOS|✅|❌[^apple-silicon]|N/A|N/A| +|macOS|✅|✅[^no-rustup]|N/A|N/A| |Windows|✅[^no-rustup]|❌|N/A|N/A| ✅: Fully supported and tested @@ -80,7 +80,6 @@ For more docs on how to build and test see [build_system/usage.txt](build_system Not all targets are available as rustup component for nightly. See notes in the platform support matrix. [^xcoff]: XCOFF object file format is not supported. -[^apple-silicon]: Tracked in [#1248](https://github.com/rust-lang/rustc_codegen_cranelift/issues/1248). [^no-rustup]: Not available as rustup component for nightly. You can build it yourself. ## Usage diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 196ff8fda75..dfbe0f51e7b 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -267,10 +267,6 @@ fn build_clif_sysroot_for_triple( prefix.to_str().unwrap() )); } - rustflags.push("-Zunstable-options".to_owned()); - for (name, values) in EXTRA_CHECK_CFGS { - rustflags.push(check_cfg_arg(name, *values)); - } compiler.rustflags.extend(rustflags); let mut build_cmd = STANDARD_LIBRARY.build(&compiler, dirs); if channel == "release" { @@ -330,34 +326,3 @@ fn build_rtstartup(dirs: &Dirs, compiler: &Compiler) -> Option { Some(target_libs) } - -// Copied from https://github.com/rust-lang/rust/blob/4fd98a4b1b100f5329c6efae18031791f64372d2/src/bootstrap/src/utils/helpers.rs#L569-L585 -/// Create a `--check-cfg` argument invocation for a given name -/// and it's values. -fn check_cfg_arg(name: &str, values: Option<&[&str]>) -> String { - // Creating a string of the values by concatenating each value: - // ',values("tvos","watchos")' or '' (nothing) when there are no values. - let next = match values { - Some(values) => { - let mut tmp = values.iter().flat_map(|val| [",", "\"", val, "\""]).collect::(); - - tmp.insert_str(1, "values("); - tmp.push(')'); - tmp - } - None => "".to_string(), - }; - format!("--check-cfg=cfg({name}{next})") -} - -const EXTRA_CHECK_CFGS: &[(&str, Option<&[&str]>)] = &[ - ("bootstrap", None), - ("stdarch_intel_sde", None), - ("no_fp_fmt_parse", None), - ("no_global_oom_handling", None), - ("no_rc", None), - ("no_sync", None), - ("netbsd10", None), - ("backtrace_in_libstd", None), - ("target_arch", Some(&["xtensa"])), -]; diff --git a/build_system/tests.rs b/build_system/tests.rs index 278f334796a..790d9cbd9fc 100644 --- a/build_system/tests.rs +++ b/build_system/tests.rs @@ -329,7 +329,6 @@ pub(crate) fn run_tests( struct TestRunner<'a> { is_native: bool, jit_supported: bool, - use_unstable_features: bool, skip_tests: &'a [&'a str], dirs: Dirs, target_compiler: Compiler, @@ -361,15 +360,7 @@ fn new( && target_compiler.triple.contains("x86_64") && !target_compiler.triple.contains("windows"); - Self { - is_native, - jit_supported, - use_unstable_features, - skip_tests, - dirs, - target_compiler, - stdlib_source, - } + Self { is_native, jit_supported, skip_tests, dirs, target_compiler, stdlib_source } } fn run_testsuite(&self, tests: &[TestCase]) { @@ -393,31 +384,13 @@ fn run_testsuite(&self, tests: &[TestCase]) { match *cmd { TestCaseCmd::Custom { func } => func(self), TestCaseCmd::BuildLib { source, crate_types } => { - if self.use_unstable_features { - self.run_rustc([source, "--crate-type", crate_types]); - } else { - self.run_rustc([ - source, - "--crate-type", - crate_types, - "--cfg", - "no_unstable_features", - ]); - } + self.run_rustc([source, "--crate-type", crate_types]); } TestCaseCmd::BuildBin { source } => { - if self.use_unstable_features { - self.run_rustc([source]); - } else { - self.run_rustc([source, "--cfg", "no_unstable_features"]); - } + self.run_rustc([source]); } TestCaseCmd::BuildBinAndRun { source, args } => { - if self.use_unstable_features { - self.run_rustc([source]); - } else { - self.run_rustc([source, "--cfg", "no_unstable_features"]); - } + self.run_rustc([source]); self.run_out_command( source.split('/').last().unwrap().split('.').next().unwrap(), args, @@ -472,7 +445,6 @@ fn rustc_command(&self, args: I) -> Command cmd.arg(&self.target_compiler.triple); cmd.arg("-Cpanic=abort"); cmd.arg("-Zunstable-options"); - cmd.arg("--check-cfg=cfg(no_unstable_features)"); cmd.arg("--check-cfg=cfg(jit)"); cmd.args(args); cmd diff --git a/example/mini_core_hello_world.rs b/example/mini_core_hello_world.rs index aab20f67248..7d361a9ab2b 100644 --- a/example/mini_core_hello_world.rs +++ b/example/mini_core_hello_world.rs @@ -333,12 +333,7 @@ struct ExternTypeWrapper { #[cfg(all(not(jit), not(all(windows, target_env = "gnu"))))] test_tls(); - #[cfg(all( - not(jit), - not(no_unstable_features), - target_arch = "x86_64", - any(target_os = "linux", target_os = "macos") - ))] + #[cfg(all(not(jit), target_arch = "x86_64", any(target_os = "linux", target_os = "macos")))] unsafe { global_asm_test(); naked_test(); @@ -367,17 +362,12 @@ fn stack_val_align() { assert_eq!(&a as *const Foo as usize % 8192, 0); } -#[cfg(all( - not(jit), - not(no_unstable_features), - target_arch = "x86_64", - any(target_os = "linux", target_os = "macos") -))] +#[cfg(all(not(jit), target_arch = "x86_64", any(target_os = "linux", target_os = "macos")))] extern "C" { fn global_asm_test(); } -#[cfg(all(not(jit), not(no_unstable_features), target_arch = "x86_64", target_os = "linux"))] +#[cfg(all(not(jit), target_arch = "x86_64", target_os = "linux"))] global_asm! { " .global global_asm_test @@ -387,7 +377,7 @@ fn stack_val_align() { " } -#[cfg(all(not(jit), not(no_unstable_features), target_arch = "x86_64", target_os = "macos"))] +#[cfg(all(not(jit), target_arch = "x86_64", target_os = "macos"))] global_asm! { " .global _global_asm_test @@ -397,7 +387,7 @@ fn stack_val_align() { " } -#[cfg(all(not(jit), not(no_unstable_features), target_arch = "x86_64"))] +#[cfg(all(not(jit), target_arch = "x86_64"))] #[naked] extern "C" fn naked_test() { unsafe { diff --git a/example/std_example.rs b/example/std_example.rs index 7347b2e7789..6cedd84adfe 100644 --- a/example/std_example.rs +++ b/example/std_example.rs @@ -251,6 +251,9 @@ unsafe fn test_simd() { test_mm_add_epi8(); test_mm_add_pd(); test_mm_cvtepi8_epi16(); + #[cfg(not(jit))] + test_mm_cvtps_epi32(); + test_mm_cvttps_epi32(); test_mm_cvtsi128_si64(); test_mm_extract_epi8(); @@ -476,6 +479,41 @@ unsafe fn test_mm256_permutevar8x32_epi32() { assert_eq_m256i(r, e); } +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +#[cfg(not(jit))] +unsafe fn test_mm_cvtps_epi32() { + let floats: [f32; 4] = [1.5, -2.5, i32::MAX as f32 + 1.0, f32::NAN]; + + let float_vec = _mm_loadu_ps(floats.as_ptr()); + let int_vec = _mm_cvtps_epi32(float_vec); + + let mut ints: [i32; 4] = [0; 4]; + _mm_storeu_si128(ints.as_mut_ptr() as *mut __m128i, int_vec); + + // this is very different from `floats.map(|f| f as i32)`! + let expected_ints: [i32; 4] = [2, -2, i32::MIN, i32::MIN]; + + assert_eq!(ints, expected_ints); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn test_mm_cvttps_epi32() { + let floats: [f32; 4] = [1.5, -2.5, i32::MAX as f32 + 1.0, f32::NAN]; + + let float_vec = _mm_loadu_ps(floats.as_ptr()); + let int_vec = _mm_cvttps_epi32(float_vec); + + let mut ints: [i32; 4] = [0; 4]; + _mm_storeu_si128(ints.as_mut_ptr() as *mut __m128i, int_vec); + + // this is very different from `floats.map(|f| f as i32)`! + let expected_ints: [i32; 4] = [1, -2, i32::MIN, i32::MIN]; + + assert_eq!(ints, expected_ints); +} + fn test_checked_mul() { let u: Option = u8::from_str_radix("1000", 10).ok(); assert_eq!(u, None); diff --git a/patches/stdlib-lock.toml b/patches/stdlib-lock.toml index c8c7b45bc9a..9ea53e8f848 100644 --- a/patches/stdlib-lock.toml +++ b/patches/stdlib-lock.toml @@ -4,12 +4,12 @@ version = 3 [[package]] name = "addr2line" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" dependencies = [ "compiler_builtins", - "gimli", + "gimli 0.29.0", "rustc-std-workspace-alloc", "rustc-std-workspace-core", ] @@ -133,6 +133,17 @@ dependencies = [ "rustc-std-workspace-core", ] +[[package]] +name = "gimli" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +dependencies = [ + "compiler_builtins", + "rustc-std-workspace-alloc", + "rustc-std-workspace-core", +] + [[package]] name = "hashbrown" version = "0.14.3" @@ -147,9 +158,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.9" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" dependencies = [ "compiler_builtins", "rustc-std-workspace-alloc", @@ -189,9 +200,9 @@ dependencies = [ [[package]] name = "object" -version = "0.32.1" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434" dependencies = [ "compiler_builtins", "memchr", @@ -286,9 +297,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" dependencies = [ "compiler_builtins", "rustc-std-workspace-core", @@ -396,8 +407,6 @@ dependencies = [ "core", "getopts", "libc", - "panic_abort", - "panic_unwind", "std", ] @@ -430,7 +439,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37a19a21a537f635c16c7576f22d0f2f7d63353c1337ad4ce0d8001c7952a25b" dependencies = [ "compiler_builtins", - "gimli", + "gimli 0.28.1", "rustc-std-workspace-core", ] diff --git a/rust-toolchain b/rust-toolchain index a2ba79cbe90..cfa91744a0e 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2024-05-13" +channel = "nightly-2024-06-30" components = ["rust-src", "rustc-dev", "llvm-tools"] diff --git a/scripts/test_rustc_tests.sh b/scripts/test_rustc_tests.sh index 689cda21643..c1b7e4b0e07 100755 --- a/scripts/test_rustc_tests.sh +++ b/scripts/test_rustc_tests.sh @@ -36,9 +36,8 @@ rm tests/ui/parser/unclosed-delimiter-in-dep.rs # submodule contains //~ERROR rm tests/ui/asm/x86_64/evex512-implicit-feature.rs # unimplemented AVX512 x86 vendor intrinsic # exotic linkages -rm tests/ui/issues/issue-33992.rs # unsupported linkages -rm tests/incremental/hashes/function_interfaces.rs # same -rm tests/incremental/hashes/statics.rs # same +rm tests/incremental/hashes/function_interfaces.rs +rm tests/incremental/hashes/statics.rs # variadic arguments rm tests/ui/abi/mir/mir_codegen_calls_variadic.rs # requires float varargs @@ -60,13 +59,20 @@ rm tests/ui/asm/x86_64/goto.rs # inline asm labels not supported # requires LTO rm -r tests/run-make/cdylib -rm -r tests/run-make/issue-14500 -rm -r tests/run-make/issue-64153 rm -r tests/run-make/codegen-options-parsing rm -r tests/run-make/lto-* rm -r tests/run-make/reproducible-build-2 rm -r tests/run-make/issue-109934-lto-debuginfo rm -r tests/run-make/no-builtins-lto +rm -r tests/run-make/reachable-extern-fn-available-lto + +# coverage instrumentation +rm tests/ui/consts/precise-drop-with-coverage.rs +rm tests/ui/issues/issue-85461.rs +rm -r tests/ui/instrument-coverage/ + +# missing f16/f128 support +rm tests/ui/half-open-range-patterns/half-open-range-pats-semantics.rs # optimization tests # ================== @@ -74,6 +80,7 @@ rm tests/ui/codegen/issue-28950.rs # depends on stack size optimizations rm tests/ui/codegen/init-large-type.rs # same rm tests/ui/issues/issue-40883.rs # same rm -r tests/run-make/fmt-write-bloat/ # tests an optimization +rm tests/ui/statics/const_generics.rs # same # backend specific tests # ====================== @@ -85,6 +92,7 @@ rm -r tests/run-make/sepcomp-cci-copies # same rm -r tests/run-make/volatile-intrinsics # same rm -r tests/run-make/llvm-ident # same rm -r tests/run-make/no-builtins-attribute # same +rm -r tests/run-make/pgo-gen-no-imp-symbols # same rm tests/ui/abi/stack-protector.rs # requires stack protector support rm -r tests/run-make/emit-stack-sizes # requires support for -Z emit-stack-sizes rm -r tests/run-make/optimization-remarks-dir # remarks are LLVM specific @@ -93,13 +101,14 @@ rm -r tests/run-make/print-to-output # requires --print relocation-models # requires asm, llvm-ir and/or llvm-bc emit support # ============================================= rm -r tests/run-make/emit-named-files -rm -r tests/run-make/issue-30063 rm -r tests/run-make/multiple-emits rm -r tests/run-make/output-type-permutations rm -r tests/run-make/emit-to-stdout rm -r tests/run-make/compressed-debuginfo rm -r tests/run-make/symbols-include-type-name - +rm -r tests/run-make/notify-all-emit-artifacts +rm -r tests/run-make/reset-codegen-1 +rm -r tests/run-make/inline-always-many-cgu # giving different but possibly correct results # ============================================= @@ -118,6 +127,7 @@ rm -r tests/run-make/compiler-builtins # Expects lib/rustlib/src/rust to contain # ============ rm -r tests/run-make/extern-fn-explicit-align # argument alignment not yet supported rm -r tests/run-make/panic-abort-eh_frame # .eh_frame emitted with panic=abort +rm tests/ui/deprecation/deprecated_inline_threshold.rs # missing deprecation warning for -Cinline-threshold # bugs in the test suite # ====================== @@ -148,12 +158,12 @@ index 9607ff02f96..b7d97caf9a2 100644 --- a/src/tools/run-make-support/src/rustdoc.rs +++ b/src/tools/run-make-support/src/rustdoc.rs @@ -34,8 +34,6 @@ pub fn bare() -> Self { - /// Construct a \`rustdoc\` invocation with \`-L \$(TARGET_RPATH_DIR)\` set. + #[track_caller] pub fn new() -> Self { let mut cmd = setup_common(); -- let target_rpath_dir = env::var_os("TARGET_RPATH_DIR").unwrap(); +- let target_rpath_dir = env_var_os("TARGET_RPATH_DIR"); - cmd.arg(format!("-L{}", target_rpath_dir.to_string_lossy())); - Self { cmd, stdin: None } + Self { cmd } } EOF diff --git a/src/abi/mod.rs b/src/abi/mod.rs index 695dbaf2804..0d7eee7afb4 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -5,8 +5,9 @@ mod returning; use std::borrow::Cow; +use std::mem; -use cranelift_codegen::ir::SigRef; +use cranelift_codegen::ir::{ArgumentPurpose, SigRef}; use cranelift_codegen::isa::CallConv; use cranelift_module::ModuleError; use rustc_codegen_ssa::errors::CompilerBuiltinsCannotCall; @@ -17,7 +18,7 @@ use rustc_monomorphize::is_call_from_compiler_builtins_to_upstream_monomorphization; use rustc_session::Session; use rustc_span::source_map::Spanned; -use rustc_target::abi::call::{Conv, FnAbi}; +use rustc_target::abi::call::{Conv, FnAbi, PassMode}; use rustc_target::spec::abi::Abi; use self::pass_mode::*; @@ -487,6 +488,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( let args = args; assert_eq!(fn_abi.args.len(), args.len()); + #[derive(Copy, Clone)] enum CallTarget { Direct(FuncRef), Indirect(SigRef, Value), @@ -532,7 +534,7 @@ enum CallTarget { }; self::returning::codegen_with_call_return_arg(fx, &fn_abi.ret, ret_place, |fx, return_ptr| { - let call_args = return_ptr + let mut call_args = return_ptr .into_iter() .chain(first_arg_override.into_iter()) .chain( @@ -545,40 +547,17 @@ enum CallTarget { ) .collect::>(); - let call_inst = match func_ref { + // FIXME: Find a cleaner way to support varargs. + if fn_abi.c_variadic { + adjust_call_for_c_variadic(fx, &fn_abi, source_info, func_ref, &mut call_args); + } + + match func_ref { CallTarget::Direct(func_ref) => fx.bcx.ins().call(func_ref, &call_args), CallTarget::Indirect(sig, func_ptr) => { fx.bcx.ins().call_indirect(sig, func_ptr, &call_args) } - }; - - // FIXME find a cleaner way to support varargs - if fn_sig.c_variadic() { - if !matches!(fn_sig.abi(), Abi::C { .. }) { - fx.tcx.dcx().span_fatal( - source_info.span, - format!("Variadic call for non-C abi {:?}", fn_sig.abi()), - ); - } - let sig_ref = fx.bcx.func.dfg.call_signature(call_inst).unwrap(); - let abi_params = call_args - .into_iter() - .map(|arg| { - let ty = fx.bcx.func.dfg.value_type(arg); - if !ty.is_int() { - // FIXME set %al to upperbound on float args once floats are supported - fx.tcx.dcx().span_fatal( - source_info.span, - format!("Non int ty {:?} for variadic call", ty), - ); - } - AbiParam::new(ty) - }) - .collect::>(); - fx.bcx.func.dfg.signatures[sig_ref].params = abi_params; } - - call_inst }); if let Some(dest) = target { @@ -587,6 +566,100 @@ enum CallTarget { } else { fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); } + + fn adjust_call_for_c_variadic<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + source_info: mir::SourceInfo, + target: CallTarget, + call_args: &mut Vec, + ) { + if fn_abi.conv != Conv::C { + fx.tcx.dcx().span_fatal( + source_info.span, + format!("Variadic call for non-C abi {:?}", fn_abi.conv), + ); + } + let sig_ref = match target { + CallTarget::Direct(func_ref) => fx.bcx.func.dfg.ext_funcs[func_ref].signature, + CallTarget::Indirect(sig_ref, _) => sig_ref, + }; + // `mem::take()` the `params` so that `fx.bcx` can be used below. + let mut abi_params = mem::take(&mut fx.bcx.func.dfg.signatures[sig_ref].params); + + // Recalculate the parameters in the signature to ensure the signature contains the variadic arguments. + let has_return_arg = matches!(fn_abi.ret.mode, PassMode::Indirect { .. }); + // Drop everything except the return argument (if there is one). + abi_params.truncate(if has_return_arg { 1 } else { 0 }); + // Add the fixed arguments. + abi_params.extend( + fn_abi.args[..fn_abi.fixed_count as usize] + .iter() + .flat_map(|arg_abi| arg_abi.get_abi_param(fx.tcx).into_iter()), + ); + let fixed_arg_count = abi_params.len(); + // Add the variadic arguments. + abi_params.extend( + fn_abi.args[fn_abi.fixed_count as usize..] + .iter() + .flat_map(|arg_abi| arg_abi.get_abi_param(fx.tcx).into_iter()), + ); + + if fx.tcx.sess.target.is_like_osx && fx.tcx.sess.target.arch == "aarch64" { + // Add any padding arguments needed for Apple AArch64. + // There's no need to pad the argument list unless variadic arguments are actually being + // passed. + if abi_params.len() > fixed_arg_count { + // 128-bit integers take 2 registers, and everything else takes 1. + // FIXME: Add support for non-integer types + // This relies on the checks below to ensure all arguments are integer types and + // that the ABI is "C". + // The return argument isn't counted as it goes in its own dedicated register. + let integer_registers_used: usize = abi_params + [if has_return_arg { 1 } else { 0 }..fixed_arg_count] + .iter() + .map(|arg| if arg.value_type.bits() == 128 { 2 } else { 1 }) + .sum(); + // The ABI uses 8 registers before it starts pushing arguments to the stack. Pad out + // the registers if needed to ensure the variadic arguments are passed on the stack. + if integer_registers_used < 8 { + abi_params.splice( + fixed_arg_count..fixed_arg_count, + (integer_registers_used..8).map(|_| AbiParam::new(types::I64)), + ); + call_args.splice( + fixed_arg_count..fixed_arg_count, + (integer_registers_used..8).map(|_| fx.bcx.ins().iconst(types::I64, 0)), + ); + } + } + + // `StructArgument` is not currently used by the `aarch64` ABI, and is therefore not + // handled when calculating how many padding arguments to use. Assert that this remains + // the case. + assert!(abi_params.iter().all(|param| matches!( + param.purpose, + // The only purposes used are `Normal` and `StructReturn`. + ArgumentPurpose::Normal | ArgumentPurpose::StructReturn + ))); + } + + // Check all parameters are integers. + for param in abi_params.iter() { + if !param.value_type.is_int() { + // FIXME: Set %al to upperbound on float args once floats are supported. + fx.tcx.dcx().span_fatal( + source_info.span, + format!("Non int ty {:?} for variadic call", param.value_type), + ); + } + } + + assert_eq!(abi_params.len(), call_args.len()); + + // Put the `AbiParam`s back in the signature. + fx.bcx.func.dfg.signatures[sig_ref].params = abi_params; + } } pub(crate) fn codegen_drop<'tcx>( diff --git a/src/allocator.rs b/src/allocator.rs index e8af3e8c255..b4a3825e996 100644 --- a/src/allocator.rs +++ b/src/allocator.rs @@ -11,15 +11,10 @@ use crate::prelude::*; /// Returns whether an allocator shim was created -pub(crate) fn codegen( - tcx: TyCtxt<'_>, - module: &mut impl Module, - unwind_context: &mut UnwindContext, -) -> bool { +pub(crate) fn codegen(tcx: TyCtxt<'_>, module: &mut dyn Module) -> bool { let Some(kind) = allocator_kind_for_codegen(tcx) else { return false }; codegen_inner( module, - unwind_context, kind, tcx.alloc_error_handler_kind(()).unwrap(), tcx.sess.opts.unstable_opts.oom, @@ -28,8 +23,7 @@ pub(crate) fn codegen( } fn codegen_inner( - module: &mut impl Module, - unwind_context: &mut UnwindContext, + module: &mut dyn Module, kind: AllocatorKind, alloc_error_handler_kind: AllocatorKind, oom_strategy: OomStrategy, @@ -67,7 +61,6 @@ fn codegen_inner( }; crate::common::create_wrapper_function( module, - unwind_context, sig, &global_fn_name(method.name), &default_fn_name(method.name), @@ -82,7 +75,6 @@ fn codegen_inner( }; crate::common::create_wrapper_function( module, - unwind_context, sig, "__rust_alloc_error_handler", &alloc_error_handler_name(alloc_error_handler_kind), diff --git a/src/base.rs b/src/base.rs index b117dc496c2..c5b4277015a 100644 --- a/src/base.rs +++ b/src/base.rs @@ -249,9 +249,7 @@ pub(crate) fn compile_fn( } // Define debuginfo for function - let isa = module.isa(); let debug_context = &mut cx.debug_context; - let unwind_context = &mut cx.unwind_context; cx.profiler.generic_activity("generate debug info").run(|| { if let Some(debug_context) = debug_context { codegened_func.func_debug_cx.unwrap().finalize( @@ -260,7 +258,6 @@ pub(crate) fn compile_fn( context, ); } - unwind_context.add_function(codegened_func.func_id, &context, isa); }); } @@ -909,7 +906,7 @@ fn is_fat_ptr<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool { | StatementKind::PlaceMention(..) | StatementKind::AscribeUserType(..) => {} - StatementKind::Coverage { .. } => fx.tcx.dcx().fatal("-Zcoverage is unimplemented"), + StatementKind::Coverage { .. } => unreachable!(), StatementKind::Intrinsic(ref intrinsic) => match &**intrinsic { // We ignore `assume` intrinsics, they are only useful for optimizations NonDivergingIntrinsic::Assume(_) => {} diff --git a/src/common.rs b/src/common.rs index 21d0cd2d30f..09317139936 100644 --- a/src/common.rs +++ b/src/common.rs @@ -247,7 +247,6 @@ pub(crate) fn type_sign(ty: Ty<'_>) -> bool { pub(crate) fn create_wrapper_function( module: &mut dyn Module, - unwind_context: &mut UnwindContext, sig: Signature, wrapper_name: &str, callee_name: &str, @@ -280,7 +279,6 @@ pub(crate) fn create_wrapper_function( bcx.finalize(); } module.define_function(wrapper_func_id, &mut ctx).unwrap(); - unwind_context.add_function(wrapper_func_id, &ctx, module.isa()); } pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> { @@ -395,6 +393,7 @@ pub(crate) fn create_stack_slot(&mut self, size: u32, align: u32) -> Pointer { // FIXME Don't force the size to a multiple of bytes once Cranelift gets // a way to specify stack slot alignment. size: (size + abi_align - 1) / abi_align * abi_align, + align_shift: 4, }); Pointer::stack_slot(stack_slot) } else { @@ -405,6 +404,7 @@ pub(crate) fn create_stack_slot(&mut self, size: u32, align: u32) -> Pointer { // FIXME Don't force the size to a multiple of bytes once Cranelift gets // a way to specify stack slot alignment. size: (size + align) / abi_align * abi_align, + align_shift: 4, }); let base_ptr = self.bcx.ins().stack_addr(self.pointer_type, stack_slot, 0); let misalign_offset = self.bcx.ins().urem_imm(base_ptr, i64::from(align)); diff --git a/src/driver/aot.rs b/src/driver/aot.rs index dcafac21bc7..763d9a48407 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -26,6 +26,7 @@ use crate::concurrency_limiter::{ConcurrencyLimiter, ConcurrencyLimiterToken}; use crate::debuginfo::TypeDebugContext; use crate::global_asm::GlobalAsmConfig; +use crate::unwind_module::UnwindModule; use crate::{prelude::*, BackendConfig}; struct ModuleCodegenResult { @@ -318,7 +319,11 @@ fn produce_final_output_artifacts( // These are used in linking steps and will be cleaned up afterward. } -fn make_module(sess: &Session, backend_config: &BackendConfig, name: String) -> ObjectModule { +fn make_module( + sess: &Session, + backend_config: &BackendConfig, + name: String, +) -> UnwindModule { let isa = crate::build_isa(sess, backend_config); let mut builder = @@ -327,16 +332,15 @@ fn make_module(sess: &Session, backend_config: &BackendConfig, name: String) -> // is important, while cg_clif cares more about compilation times. Enabling -Zfunction-sections // can easily double the amount of time necessary to perform linking. builder.per_function_section(sess.opts.unstable_opts.function_sections.unwrap_or(false)); - ObjectModule::new(builder) + UnwindModule::new(ObjectModule::new(builder), true) } fn emit_cgu( output_filenames: &OutputFilenames, prof: &SelfProfilerRef, name: String, - module: ObjectModule, + module: UnwindModule, debug: Option, - unwind_context: UnwindContext, global_asm_object_file: Option, producer: &str, ) -> Result { @@ -346,8 +350,6 @@ fn emit_cgu( debug.emit(&mut product); } - unwind_context.emit(&mut product); - let module_regular = emit_module( output_filenames, prof, @@ -494,7 +496,6 @@ fn module_codegen( let mut cx = crate::CodegenCx::new( tcx, - backend_config.clone(), module.isa(), tcx.sess.opts.debuginfo != DebugInfo::None, cgu_name, @@ -531,13 +532,7 @@ fn module_codegen( } } } - crate::main_shim::maybe_create_entry_wrapper( - tcx, - &mut module, - &mut cx.unwind_context, - false, - cgu.is_primary(), - ); + crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, false, cgu.is_primary()); let cgu_name = cgu.name().as_str().to_owned(); @@ -571,7 +566,6 @@ fn module_codegen( cgu_name, module, cx.debug_context, - cx.unwind_context, global_asm_object_file, &producer, ) @@ -665,13 +659,10 @@ pub(crate) fn run_aot( }); let mut allocator_module = make_module(tcx.sess, &backend_config, "allocator_shim".to_string()); - let mut allocator_unwind_context = UnwindContext::new(allocator_module.isa(), true); - let created_alloc_shim = - crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context); + let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module); let allocator_module = if created_alloc_shim { - let mut product = allocator_module.finish(); - allocator_unwind_context.emit(&mut product); + let product = allocator_module.finish(); match emit_module( tcx.output_filenames(()), diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 4b149131b61..dfee8e714e6 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -14,12 +14,12 @@ use rustc_span::Symbol; use crate::debuginfo::TypeDebugContext; +use crate::unwind_module::UnwindModule; use crate::{prelude::*, BackendConfig}; use crate::{CodegenCx, CodegenMode}; struct JitState { - backend_config: BackendConfig, - jit_module: JITModule, + jit_module: UnwindModule, } thread_local! { @@ -63,7 +63,7 @@ fn create_jit_module( tcx: TyCtxt<'_>, backend_config: &BackendConfig, hotswap: bool, -) -> (JITModule, CodegenCx) { +) -> (UnwindModule, CodegenCx) { let crate_info = CrateInfo::new(tcx, "dummy_target_cpu".to_string()); let isa = crate::build_isa(tcx.sess, backend_config); @@ -72,17 +72,11 @@ fn create_jit_module( crate::compiler_builtins::register_functions_for_jit(&mut jit_builder); jit_builder.symbol_lookup_fn(dep_symbol_lookup_fn(tcx.sess, crate_info)); jit_builder.symbol("__clif_jit_fn", clif_jit_fn as *const u8); - let mut jit_module = JITModule::new(jit_builder); + let mut jit_module = UnwindModule::new(JITModule::new(jit_builder), false); - let mut cx = crate::CodegenCx::new( - tcx, - backend_config.clone(), - jit_module.isa(), - false, - Symbol::intern("dummy_cgu_name"), - ); + let cx = crate::CodegenCx::new(tcx, jit_module.isa(), false, Symbol::intern("dummy_cgu_name")); - crate::allocator::codegen(tcx, &mut jit_module, &mut cx.unwind_context); + crate::allocator::codegen(tcx, &mut jit_module); (jit_module, cx) } @@ -128,7 +122,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { ); } CodegenMode::JitLazy => { - codegen_shim(tcx, &mut cx, &mut cached_context, &mut jit_module, inst) + codegen_shim(tcx, &mut cached_context, &mut jit_module, inst) } }, MonoItem::Static(def_id) => { @@ -146,18 +140,11 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { tcx.dcx().fatal("Inline asm is not supported in JIT mode"); } - crate::main_shim::maybe_create_entry_wrapper( - tcx, - &mut jit_module, - &mut cx.unwind_context, - true, - true, - ); + crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, true, true); tcx.dcx().abort_if_errors(); - jit_module.finalize_definitions().unwrap(); - unsafe { cx.unwind_context.register_jit(&jit_module) }; + jit_module.finalize_definitions(); println!( "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed" @@ -177,12 +164,12 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { call_conv: jit_module.target_config().default_call_conv, }; let start_func_id = jit_module.declare_function("main", Linkage::Import, &start_sig).unwrap(); - let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id); + let finalized_start: *const u8 = jit_module.module.get_finalized_function(start_func_id); LAZY_JIT_STATE.with(|lazy_jit_state| { let mut lazy_jit_state = lazy_jit_state.borrow_mut(); assert!(lazy_jit_state.is_none()); - *lazy_jit_state = Some(JitState { backend_config, jit_module }); + *lazy_jit_state = Some(JitState { jit_module }); }); let f: extern "C" fn(c_int, *const *const c_char) -> c_int = @@ -268,7 +255,6 @@ fn jit_fn(instance_ptr: *const Instance<'static>, trampoline_ptr: *const u8) -> let mut lazy_jit_state = lazy_jit_state.borrow_mut(); let lazy_jit_state = lazy_jit_state.as_mut().unwrap(); let jit_module = &mut lazy_jit_state.jit_module; - let backend_config = lazy_jit_state.backend_config.clone(); let name = tcx.symbol_name(instance).name; let sig = crate::abi::get_function_sig( @@ -278,7 +264,7 @@ fn jit_fn(instance_ptr: *const Instance<'static>, trampoline_ptr: *const u8) -> ); let func_id = jit_module.declare_function(name, Linkage::Export, &sig).unwrap(); - let current_ptr = jit_module.read_got_entry(func_id); + let current_ptr = jit_module.module.read_got_entry(func_id); // If the function's GOT entry has already been updated to point at something other // than the shim trampoline, don't re-jit but just return the new pointer instead. @@ -288,11 +274,10 @@ fn jit_fn(instance_ptr: *const Instance<'static>, trampoline_ptr: *const u8) -> return current_ptr; } - jit_module.prepare_for_function_redefine(func_id).unwrap(); + jit_module.module.prepare_for_function_redefine(func_id).unwrap(); let mut cx = crate::CodegenCx::new( tcx, - backend_config, jit_module.isa(), false, Symbol::intern("dummy_cgu_name"), @@ -300,9 +285,8 @@ fn jit_fn(instance_ptr: *const Instance<'static>, trampoline_ptr: *const u8) -> codegen_and_compile_fn(tcx, &mut cx, &mut Context::new(), jit_module, instance); assert!(cx.global_asm.is_empty()); - jit_module.finalize_definitions().unwrap(); - unsafe { cx.unwind_context.register_jit(&jit_module) }; - jit_module.get_finalized_function(func_id) + jit_module.finalize_definitions(); + jit_module.module.get_finalized_function(func_id) }) }) } @@ -310,7 +294,7 @@ fn jit_fn(instance_ptr: *const Instance<'static>, trampoline_ptr: *const u8) -> fn dep_symbol_lookup_fn( sess: &Session, crate_info: CrateInfo, -) -> Box Option<*const u8>> { +) -> Box Option<*const u8> + Send> { use rustc_middle::middle::dependency_format::Linkage; let mut dylib_paths = Vec::new(); @@ -362,9 +346,8 @@ fn dep_symbol_lookup_fn( fn codegen_shim<'tcx>( tcx: TyCtxt<'tcx>, - cx: &mut CodegenCx, cached_context: &mut Context, - module: &mut JITModule, + module: &mut UnwindModule, inst: Instance<'tcx>, ) { let pointer_type = module.target_config().pointer_type(); @@ -413,5 +396,4 @@ fn codegen_shim<'tcx>( trampoline_builder.ins().return_(&ret_vals); module.define_function(func_id, context).unwrap(); - cx.unwind_context.add_function(func_id, context, module.isa()); } diff --git a/src/inline_asm.rs b/src/inline_asm.rs index c6b26dd873b..c88230c9360 100644 --- a/src/inline_asm.rs +++ b/src/inline_asm.rs @@ -113,13 +113,7 @@ pub(crate) fn codegen_inline_asm_terminator<'tcx>( ); let sig = get_function_sig(fx.tcx, fx.target_config.default_call_conv, instance); - create_wrapper_function( - fx.module, - &mut fx.cx.unwind_context, - sig, - &wrapper_name, - symbol.name, - ); + create_wrapper_function(fx.module, sig, &wrapper_name, symbol.name); CInlineAsmOperand::Symbol { symbol: wrapper_name } } else { @@ -283,13 +277,7 @@ pub(crate) fn codegen_naked_asm<'tcx>( ); let sig = get_function_sig(tcx, module.target_config().default_call_conv, instance); - create_wrapper_function( - module, - &mut cx.unwind_context, - sig, - &wrapper_name, - symbol.name, - ); + create_wrapper_function(module, sig, &wrapper_name, symbol.name); CInlineAsmOperand::Symbol { symbol: wrapper_name } } else { diff --git a/src/intrinsics/llvm_x86.rs b/src/intrinsics/llvm_x86.rs index d454f3c1de7..399518e58d8 100644 --- a/src/intrinsics/llvm_x86.rs +++ b/src/intrinsics/llvm_x86.rs @@ -459,11 +459,20 @@ fn select4( intrinsic_args!(fx, args => (a); intrinsic); let a = a.load_scalar(fx); + let value = fx.bcx.ins().x86_cvtt2dq(types::I32X4, a); + let cvalue = CValue::by_val(value, ret.layout()); + ret.write_cvalue(fx, cvalue); + } + "llvm.x86.sse2.cvtps2dq" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtps_epi32 + intrinsic_args!(fx, args => (a); intrinsic); + let a = a.load_scalar(fx); + // Using inline asm instead of fcvt_to_sint_sat as unrepresentable values are turned // into 0x80000000 for which Cranelift doesn't have a native instruction. codegen_inline_asm_inner( fx, - &[InlineAsmTemplatePiece::String(format!("cvttps2dq xmm0, xmm0"))], + &[InlineAsmTemplatePiece::String(format!("cvtps2dq xmm0, xmm0"))], &[CInlineAsmOperand::InOut { reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), _late: true, @@ -1416,6 +1425,36 @@ fn select4( ret.write_cvalue(fx, res); } + "llvm.x86.rdtsc" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_rdtsc&ig_expand=5273 + + let res_place = CPlace::new_stack_slot( + fx, + fx.layout_of(Ty::new_tup(fx.tcx, &[fx.tcx.types.u32, fx.tcx.types.u32])), + ); + let eax_place = res_place.place_field(fx, FieldIdx::new(0)); + let edx_place = res_place.place_field(fx, FieldIdx::new(1)); + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("rdtsc".to_string())], + &[ + CInlineAsmOperand::Out { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)), + late: true, + place: Some(eax_place), + }, + CInlineAsmOperand::Out { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)), + late: true, + place: Some(edx_place), + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::NOMEM, + ); + let res = res_place.to_cvalue(fx); + ret.write_cvalue_transmute(fx, res); + } + _ => { fx.tcx .dcx() diff --git a/src/lib.rs b/src/lib.rs index 06ca52b3903..192e6c91ea3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -79,6 +79,7 @@ mod toolchain; mod trap; mod unsize; +mod unwind_module; mod value_and_place; mod vtable; @@ -130,22 +131,13 @@ struct CodegenCx { global_asm: String, inline_asm_index: Cell, debug_context: Option, - unwind_context: UnwindContext, cgu_name: Symbol, } impl CodegenCx { - fn new( - tcx: TyCtxt<'_>, - backend_config: BackendConfig, - isa: &dyn TargetIsa, - debug_info: bool, - cgu_name: Symbol, - ) -> Self { + fn new(tcx: TyCtxt<'_>, isa: &dyn TargetIsa, debug_info: bool, cgu_name: Symbol) -> Self { assert_eq!(pointer_ty(tcx), isa.pointer_type()); - let unwind_context = - UnwindContext::new(isa, matches!(backend_config.codegen_mode, CodegenMode::Aot)); let debug_context = if debug_info && !tcx.sess.target.options.is_like_windows { Some(DebugContext::new(tcx, isa, cgu_name.as_str())) } else { @@ -158,7 +150,6 @@ fn new( global_asm: String::new(), inline_asm_index: Cell::new(0), debug_context, - unwind_context, cgu_name, } } @@ -175,7 +166,7 @@ fn locale_resource(&self) -> &'static str { } fn init(&self, sess: &Session) { - use rustc_session::config::Lto; + use rustc_session::config::{InstrumentCoverage, Lto}; match sess.lto() { Lto::No | Lto::ThinLocal => {} Lto::Thin | Lto::Fat => { @@ -183,6 +174,11 @@ fn init(&self, sess: &Session) { } } + if sess.opts.cg.instrument_coverage() != InstrumentCoverage::No { + sess.dcx() + .fatal("`-Cinstrument-coverage` is LLVM specific and not supported by Cranelift"); + } + let mut config = self.config.borrow_mut(); if config.is_none() { let new_config = BackendConfig::from_opts(&sess.opts.cg.llvm_args) diff --git a/src/main_shim.rs b/src/main_shim.rs index f9a729618a5..33d3f9b8a90 100644 --- a/src/main_shim.rs +++ b/src/main_shim.rs @@ -11,8 +11,7 @@ /// users main function. pub(crate) fn maybe_create_entry_wrapper( tcx: TyCtxt<'_>, - module: &mut impl Module, - unwind_context: &mut UnwindContext, + module: &mut dyn Module, is_jit: bool, is_primary_cgu: bool, ) { @@ -36,12 +35,11 @@ pub(crate) fn maybe_create_entry_wrapper( return; } - create_entry_fn(tcx, module, unwind_context, main_def_id, is_jit, is_main_fn, sigpipe); + create_entry_fn(tcx, module, main_def_id, is_jit, is_main_fn, sigpipe); fn create_entry_fn( tcx: TyCtxt<'_>, - m: &mut impl Module, - unwind_context: &mut UnwindContext, + m: &mut dyn Module, rust_main_def_id: DefId, ignore_lang_start_wrapper: bool, is_main_fn: bool, @@ -170,7 +168,5 @@ fn create_entry_fn( if let Err(err) = m.define_function(cmain_func_id, &mut ctx) { tcx.dcx().fatal(format!("entry symbol `{entry_name}` defined multiple times: {err}")); } - - unwind_context.add_function(cmain_func_id, &ctx, m.isa()); } } diff --git a/src/unwind_module.rs b/src/unwind_module.rs new file mode 100644 index 00000000000..b950aaa29ce --- /dev/null +++ b/src/unwind_module.rs @@ -0,0 +1,115 @@ +use cranelift_codegen::control::ControlPlane; +use cranelift_codegen::ir::{Function, Signature}; +use cranelift_codegen::isa::{TargetFrontendConfig, TargetIsa}; +use cranelift_codegen::{Context, FinalizedMachReloc}; +use cranelift_module::{ + DataDescription, DataId, FuncId, FuncOrDataId, Linkage, Module, ModuleDeclarations, + ModuleResult, +}; +use cranelift_object::{ObjectModule, ObjectProduct}; + +use crate::UnwindContext; + +/// A wrapper around a [Module] which adds any defined function to the [UnwindContext]. +pub(crate) struct UnwindModule { + pub(crate) module: T, + unwind_context: UnwindContext, +} + +impl UnwindModule { + pub(crate) fn new(module: T, pic_eh_frame: bool) -> Self { + let unwind_context = UnwindContext::new(module.isa(), pic_eh_frame); + UnwindModule { module, unwind_context } + } +} + +impl UnwindModule { + pub(crate) fn finish(self) -> ObjectProduct { + let mut product = self.module.finish(); + self.unwind_context.emit(&mut product); + product + } +} + +#[cfg(feature = "jit")] +impl UnwindModule { + pub(crate) fn finalize_definitions(&mut self) { + self.module.finalize_definitions().unwrap(); + let prev_unwind_context = std::mem::replace( + &mut self.unwind_context, + UnwindContext::new(self.module.isa(), false), + ); + unsafe { prev_unwind_context.register_jit(&self.module) }; + } +} + +impl Module for UnwindModule { + fn isa(&self) -> &dyn TargetIsa { + self.module.isa() + } + + fn declarations(&self) -> &ModuleDeclarations { + self.module.declarations() + } + + fn get_name(&self, name: &str) -> Option { + self.module.get_name(name) + } + + fn target_config(&self) -> TargetFrontendConfig { + self.module.target_config() + } + + fn declare_function( + &mut self, + name: &str, + linkage: Linkage, + signature: &Signature, + ) -> ModuleResult { + self.module.declare_function(name, linkage, signature) + } + + fn declare_anonymous_function(&mut self, signature: &Signature) -> ModuleResult { + self.module.declare_anonymous_function(signature) + } + + fn declare_data( + &mut self, + name: &str, + linkage: Linkage, + writable: bool, + tls: bool, + ) -> ModuleResult { + self.module.declare_data(name, linkage, writable, tls) + } + + fn declare_anonymous_data(&mut self, writable: bool, tls: bool) -> ModuleResult { + self.module.declare_anonymous_data(writable, tls) + } + + fn define_function_with_control_plane( + &mut self, + func: FuncId, + ctx: &mut Context, + ctrl_plane: &mut ControlPlane, + ) -> ModuleResult<()> { + self.module.define_function_with_control_plane(func, ctx, ctrl_plane)?; + self.unwind_context.add_function(func, ctx, self.module.isa()); + Ok(()) + } + + fn define_function_bytes( + &mut self, + _func_id: FuncId, + _func: &Function, + _alignment: u64, + _bytes: &[u8], + _relocs: &[FinalizedMachReloc], + ) -> ModuleResult<()> { + unimplemented!() + } + + fn define_data(&mut self, data_id: DataId, data: &DataDescription) -> ModuleResult<()> { + self.module.define_data(data_id, data) + } +}