diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 3f9e779dfc1..d181bd1604e 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -186,10 +186,6 @@ jobs:
       run: |
         # WIP Disable some tests
 
-        # This fails due to some weird argument handling by hyperfine, not an actual regression
-        # more of a build system issue
-        (Get-Content config.txt) -replace '(bench.simple-raytracer)', '# $1' |  Out-File config.txt
-
         # This fails with a different output than expected
         (Get-Content config.txt) -replace '(test.regex-shootout-regex-dna)', '# $1' |  Out-File config.txt
 
diff --git a/build_system/tests.rs b/build_system/tests.rs
index 81993b9d337..37c35106af6 100644
--- a/build_system/tests.rs
+++ b/build_system/tests.rs
@@ -2,7 +2,7 @@ use super::build_sysroot;
 use super::config;
 use super::path::{Dirs, RelPath};
 use super::prepare::GitRepo;
-use super::rustc_info::get_wrapper_file_name;
+use super::rustc_info::{get_file_name, get_wrapper_file_name};
 use super::utils::{
     hyperfine_command, is_ci, spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler,
 };
@@ -318,13 +318,20 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
 
             eprintln!("[BENCH RUN] ebobby/simple-raytracer");
             fs::copy(
-                target_dir.join("debug").join("main"),
-                RelPath::BUILD.to_path(&runner.dirs).join("raytracer_cg_clif"),
+                target_dir.join("debug").join(get_file_name("main", "bin")),
+                RelPath::BUILD
+                    .to_path(&runner.dirs)
+                    .join(get_file_name("raytracer_cg_clif", "bin")),
             )
             .unwrap();
 
-            let mut bench_run =
-                hyperfine_command(0, run_runs, None, "./raytracer_cg_llvm", "./raytracer_cg_clif");
+            let mut bench_run = hyperfine_command(
+                0,
+                run_runs,
+                None,
+                Path::new(".").join(get_file_name("raytracer_cg_llvm", "bin")).to_str().unwrap(),
+                Path::new(".").join(get_file_name("raytracer_cg_clif", "bin")).to_str().unwrap(),
+            );
             bench_run.current_dir(RelPath::BUILD.to_path(&runner.dirs));
             spawn_and_wait(bench_run);
         } else {