mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Test the PGO/BOLT/LTO optimized x64 Linux compiler on CI

Browse Source
This commit is contained in:
Jakub Beránek 2023-06-05 21:54:52 +02:00
parent 774a3d1523
commit c91a3a4d0c
No known key found for this signature in database
GPG Key ID: 909CD0D26483516B
3 changed files with 127 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/bootstrap/lib.rs
View File

@ -222,6 +222,7 @@ pub struct Build {
initial_cargo: PathBuf,
initial_lld: PathBuf,
initial_libdir: PathBuf,
initial_sysroot: PathBuf,
// Runtime state filled in later on
// C/C++ compilers and archiver for all targets
@ -389,13 +390,16 @@ pub fn new(mut config: Config) -> Build {
"/dummy".to_string()
} else {
output(Command::new(&config.initial_rustc).arg("--print").arg("sysroot"))
};
}
.trim()
.to_string();
let initial_libdir = initial_target_dir
.parent()
.unwrap()
.parent()
.unwrap()
.strip_prefix(initial_sysroot.trim())
.strip_prefix(&initial_sysroot)
.unwrap()
.to_path_buf();
@ -425,6 +429,7 @@ pub fn new(mut config: Config) -> Build {
initial_cargo: config.initial_cargo.clone(),
initial_lld,
initial_libdir,
initial_sysroot: initial_sysroot.into(),
local_rebuild: config.local_rebuild,
fail_fast: config.cmd.fail_fast(),
doc_tests: config.cmd.doc_tests(),

10
src/bootstrap/test.rs
View File

@ -1424,7 +1424,15 @@ fn run(self, builder: &Builder<'_>) {
cmd.arg("--src-base").arg(builder.src.join("tests").join(suite));
cmd.arg("--build-base").arg(testdir(builder, compiler.host).join(suite));
cmd.arg("--sysroot-base").arg(builder.sysroot(compiler));
// When top stage is 0, that means that we're testing an externally provided compiler.
// In that case we need to use its specific sysroot for tests to pass.
let sysroot = if builder.top_stage == 0 {
builder.initial_sysroot.clone()
} else {
builder.sysroot(compiler).to_path_buf()
};
cmd.arg("--sysroot-base").arg(sysroot);
cmd.arg("--stage-id").arg(stage_id);
cmd.arg("--suite").arg(suite);
cmd.arg("--mode").arg(mode);

115
src/ci/stage-build.py
View File

@ -124,6 +124,12 @@ class Pipeline:
def metrics_path(self) -> Path:
return self.build_root() / "build" / "metrics.json"
def executable_extension(self) -> str:
raise NotImplementedError
def skipped_tests(self) -> Iterable[str]:
return ()
class LinuxPipeline(Pipeline):
def checkout_path(self) -> Path:
@ -152,6 +158,13 @@ class LinuxPipeline(Pipeline):
def supports_bolt(self) -> bool:
return True
def executable_extension(self) -> str:
return ""
def skipped_tests(self) -> Iterable[str]:
# This test fails because of linker errors, as of June 2023.
yield "tests/ui/process/nofile-limit.rs"
class WindowsPipeline(Pipeline):
def __init__(self):
@ -211,6 +224,13 @@ class WindowsPipeline(Pipeline):
def supports_bolt(self) -> bool:
return False
def executable_extension(self) -> str:
return ".exe"
def skipped_tests(self) -> Iterable[str]:
# This test fails as of June 2023
yield "tests\\codegen\\vec-shrink-panik.rs"
def get_timestamp() -> float:
return time.time()
@ -403,9 +423,9 @@ def delete_directory(path: Path):
shutil.rmtree(path)
def unpack_archive(archive: Path):
def unpack_archive(archive: Path, target_dir: Optional[Path] = None):
LOGGER.info(f"Unpacking archive `{archive}`")
shutil.unpack_archive(archive)
shutil.unpack_archive(str(archive), extract_dir=str(target_dir) if target_dir is not None else None)
def download_file(src: str, target: Path):
@ -455,6 +475,7 @@ def cmd(
)
return subprocess.run(args, env=environment, check=True)
class BenchmarkRunner:
def run_rustc(self, pipeline: Pipeline):
raise NotImplementedError
@ -465,6 +486,7 @@ class BenchmarkRunner:
def run_bolt(self, pipeline: Pipeline):
raise NotImplementedError
class DefaultBenchmarkRunner(BenchmarkRunner):
def run_rustc(self, pipeline: Pipeline):
# Here we're profiling the `rustc` frontend, so we also include `Check`.
@ -478,6 +500,7 @@ class DefaultBenchmarkRunner(BenchmarkRunner):
LLVM_PROFILE_FILE=str(pipeline.rustc_profile_template_path())
)
)
def run_llvm(self, pipeline: Pipeline):
run_compiler_benchmarks(
pipeline,
@ -494,6 +517,7 @@ class DefaultBenchmarkRunner(BenchmarkRunner):
crates=LLVM_BOLT_CRATES
)
def run_compiler_benchmarks(
pipeline: Pipeline,
profiles: List[str],
@ -650,10 +674,8 @@ def gather_llvm_profiles(pipeline: Pipeline, runner: BenchmarkRunner):
def gather_rustc_profiles(pipeline: Pipeline, runner: BenchmarkRunner):
LOGGER.info("Running benchmarks with PGO instrumented rustc")
runner.run_rustc(pipeline)
profile_path = pipeline.rustc_profile_merged_file()
LOGGER.info(f"Merging Rustc PGO profiles to {profile_path}")
cmd([
@ -770,6 +792,86 @@ def record_metrics(pipeline: Pipeline, timer: Timer):
log_metrics(metrics)
def run_tests(pipeline: Pipeline):
"""
After `dist` is executed, we extract its archived components into a sysroot directory,
and then use that extracted rustc as a stage0 compiler.
Then we run a subset of tests using that compiler, to have a basic smoke test which checks
whether the optimization pipeline hasn't broken something.
"""
build_dir = pipeline.build_root() / "build"
dist_dir = build_dir / "dist"
def extract_dist_dir(name: str) -> Path:
target_dir = build_dir / "optimized-dist"
target_dir.mkdir(parents=True, exist_ok=True)
unpack_archive(dist_dir / f"{name}.tar.xz", target_dir=target_dir)
extracted_path = target_dir / name
assert extracted_path.is_dir()
return extracted_path
# Extract rustc, libstd, cargo and src archives to create the optimized sysroot
rustc_dir = extract_dist_dir(f"rustc-nightly-{PGO_HOST}") / "rustc"
libstd_dir = extract_dist_dir(f"rust-std-nightly-{PGO_HOST}") / f"rust-std-{PGO_HOST}"
cargo_dir = extract_dist_dir(f"cargo-nightly-{PGO_HOST}") / f"cargo"
extracted_src_dir = extract_dist_dir("rust-src-nightly") / "rust-src"
# We need to manually copy libstd to the extracted rustc sysroot
shutil.copytree(
libstd_dir / "lib" / "rustlib" / PGO_HOST / "lib",
rustc_dir / "lib" / "rustlib" / PGO_HOST / "lib"
)
# Extract sources - they aren't in the `rustc-nightly-{host}` tarball, so we need to manually copy libstd
# sources to the extracted sysroot. We need sources available so that `-Zsimulate-remapped-rust-src-base`
# works correctly.
shutil.copytree(
extracted_src_dir / "lib" / "rustlib" / "src",
rustc_dir / "lib" / "rustlib" / "src"
)
rustc_path = rustc_dir / "bin" / f"rustc{pipeline.executable_extension()}"
assert rustc_path.is_file()
cargo_path = cargo_dir / "bin" / f"cargo{pipeline.executable_extension()}"
assert cargo_path.is_file()
config_content = f"""profile = "user"
changelog-seen = 2
[build]
rustc = "{rustc_path.as_posix()}"
cargo = "{cargo_path.as_posix()}"
[llvm]
download-ci-llvm = true
"""
logging.info(f"Using following `config.toml` for running tests:\n{config_content}")
# Simulate a stage 0 compiler with the extracted optimized dist artifacts.
with open("config.toml", "w") as f:
f.write(config_content)
args = [
sys.executable,
pipeline.checkout_path() / "x.py",
"test",
"--stage", "0",
"tests/assembly",
"tests/codegen",
"tests/codegen-units",
"tests/incremental",
"tests/mir-opt",
"tests/pretty",
"tests/run-pass-valgrind",
"tests/ui",
]
for test_path in pipeline.skipped_tests():
args.extend(["--exclude", test_path])
cmd(args=args, env=dict(
COMPILETEST_FORCE_STAGE0="1"
))
def execute_build_pipeline(timer: Timer, pipeline: Pipeline, runner: BenchmarkRunner, final_build_args: List[str]):
# Clear and prepare tmp directory
shutil.rmtree(pipeline.opt_artifacts(), ignore_errors=True)
@ -844,6 +946,11 @@ def execute_build_pipeline(timer: Timer, pipeline: Pipeline, runner: BenchmarkRu
cmd(final_build_args)
record_metrics(pipeline, stage4)
# Try builds can be in various broken states, so we don't want to gatekeep them with tests
if not is_try_build():
with timer.section("Run tests"):
run_tests(pipeline)
def run(runner: BenchmarkRunner):
logging.basicConfig(