mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

miri-script: start and stop josh automatically

Browse Source
This commit is contained in:
Ralf Jung 2023-08-22 10:57:09 +02:00
parent 6117fa1fef
commit ee1fbd7a93
7 changed files with 212 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/tools/miri/.github/workflows/ci.yml vendored
View File

@ -189,8 +189,6 @@ jobs:
fetch-depth: 256 # get a bit more of the history
- name: install josh-proxy
run: cargo +stable install josh-proxy --git https://github.com/josh-project/josh --tag r22.12.06
- name: start josh-proxy
run: josh-proxy --local=$HOME/.cache/josh --remote=https://github.com --no-background &
- name: setup bot git name and email
run: |
git config --global user.name 'The Miri Conjob Bot'

37
src/tools/miri/CONTRIBUTING.md
View File

@ -231,25 +231,16 @@ You can also directly run Miri on a Rust source file:
## Advanced topic: Syncing with the rustc repo
We use the [`josh` proxy](https://github.com/josh-project/josh) to transmit changes between the
rustc and Miri repositories.
rustc and Miri repositories. You can install it as follows:
```sh
cargo +stable install josh-proxy --git https://github.com/josh-project/josh --tag r22.12.06
josh-proxy --local=$HOME/.cache/josh --remote=https://github.com --no-background
```
This uses a directory `$HOME/.cache/josh` as a cache, to speed up repeated pulling/pushing.
To make josh push via ssh instead of https, you can add the following to your `.gitconfig`:
```toml
[url "git@github.com:"]
pushInsteadOf = https://github.com/
```
Josh will automatically be started and stopped by `./miri`.
### Importing changes from the rustc repo
Josh needs to be running, as described above.
We assume we start on an up-to-date master branch in the Miri repo.
```sh
@ -268,16 +259,14 @@ needed.
### Exporting changes to the rustc repo
Keep in mind that pushing is the most complicated job that josh has to do --
pulling just filters the rustc history, but pushing needs to construct a new
rustc history that would filter to the given Miri history! To avoid problems, it
is a good idea to always pull immediately before you push. In particular, you
should never do two josh pushes without an intermediate pull; that can lead to
duplicated commits.
Keep in mind that pushing is the most complicated job that josh has to do -- pulling just filters
the rustc history, but pushing needs to construct a new rustc history that would filter to the given
Miri history! To avoid problems, it is a good idea to always pull immediately before you push. If
you are getting strange errors, chances are you are running into [this josh
bug](https://github.com/josh-project/josh/issues/998). In that case, please get in touch on Zulip.
Josh needs to be running, as described above. We will use the josh proxy to push
to your fork of rustc. Run the following in the Miri repo, assuming we are on an
up-to-date master branch:
We will use the josh proxy to push to your fork of rustc. Run the following in the Miri repo,
assuming we are on an up-to-date master branch:
```sh
# Push the Miri changes to your rustc fork (substitute your github handle for YOUR_NAME).
@ -287,3 +276,11 @@ up-to-date master branch:
This will create a new branch called 'miri' in your fork, and the output should
include a link to create a rustc PR that will integrate those changes into the
main repository.
If this fails due to authentication problems, it can help to make josh push via ssh instead of
https. Add the following to your `.gitconfig`:
```toml
[url "git@github.com:"]
pushInsteadOf = https://github.com/
```

125
src/tools/miri/miri-script/Cargo.lock
View File

@ -8,6 +8,38 @@ version = "1.0.71"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "directories"
version = "4.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f51c5d4ddabd36886dd3e1438cb358cdcb0d7c499cb99cb4ac2e38e18b5cb210"
dependencies = [
"dirs-sys",
]
[[package]]
name = "dirs-sys"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
dependencies = [
"libc",
"redox_users",
"winapi",
]
[[package]]
name = "dunce"
version = "1.0.4"
@ -20,6 +52,17 @@ version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
[[package]]
name = "getrandom"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
dependencies = [
"cfg-if",
"libc",
"wasi",
]
[[package]]
name = "itertools"
version = "0.10.5"
@ -40,6 +83,7 @@ name = "miri-script"
version = "0.1.0"
dependencies = [
"anyhow",
"directories",
"dunce",
"itertools",
"path_macro",
@ -62,6 +106,44 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6e819bbd49d5939f682638fa54826bf1650abddcd65d000923de8ad63cc7d15"
[[package]]
name = "proc-macro2"
version = "1.0.66"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
dependencies = [
"proc-macro2",
]
[[package]]
name = "redox_syscall"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
dependencies = [
"bitflags",
]
[[package]]
name = "redox_users"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
dependencies = [
"getrandom",
"redox_syscall",
"thiserror",
]
[[package]]
name = "rustc_version"
version = "0.4.0"
@ -92,6 +174,43 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde"
[[package]]
name = "syn"
version = "2.0.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "thiserror"
version = "1.0.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "unicode-ident"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c"
[[package]]
name = "walkdir"
version = "2.3.3"
@ -102,6 +221,12 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "which"
version = "4.4.0"

1
src/tools/miri/miri-script/Cargo.toml
View File

@ -20,3 +20,4 @@ anyhow = "1.0"
xshell = "0.2"
rustc_version = "0.4"
dunce = "1.0.4"
directories = "4"

69
src/tools/miri/miri-script/src/commands.rs
View File

@ -2,6 +2,9 @@
use std::ffi::OsString;
use std::io::Write;
use std::ops::Not;
use std::process;
use std::thread;
use std::time;
use anyhow::{anyhow, bail, Context, Result};
use path_macro::path;
@ -14,6 +17,7 @@
/// Used for rustc syncs.
const JOSH_FILTER: &str =
":rev(75dd959a3a40eb5b4574f8d2e23aa6efbeb33573:prefix=src/tools/miri):/src/tools/miri";
const JOSH_PORT: &str = "42042";
impl MiriEnv {
fn build_miri_sysroot(&mut self, quiet: bool) -> Result<()> {
@ -81,6 +85,55 @@ fn auto_actions() -> Result<()> {
Ok(())
}
fn start_josh() -> Result<impl Drop> {
// Determine cache directory.
let local_dir = {
let user_dirs =
directories::ProjectDirs::from("org", "rust-lang", "miri-josh").unwrap();
user_dirs.cache_dir().to_owned()
};
// Start josh, silencing its output.
let mut cmd = process::Command::new("josh-proxy");
cmd.arg("--local").arg(local_dir);
cmd.arg("--remote").arg("https://github.com");
cmd.arg("--port").arg(JOSH_PORT);
cmd.arg("--no-background");
cmd.stdout(process::Stdio::null());
cmd.stderr(process::Stdio::null());
let josh = cmd.spawn().context("failed to start josh-proxy, make sure it is installed")?;
// Give it some time so hopefully the port is open. (10ms was not enough.)
thread::sleep(time::Duration::from_millis(100));
// Create a wrapper that stops it on drop.
struct Josh(process::Child);
impl Drop for Josh {
fn drop(&mut self) {
#[cfg(unix)]
{
// Try to gracefully shut it down.
process::Command::new("kill")
.args(["-s", "INT", &self.0.id().to_string()])
.output()
.expect("failed to SIGINT josh-proxy");
// Sadly there is no "wait with timeout"... so we just give it some time to finish.
thread::sleep(time::Duration::from_millis(100));
// Now hopefully it is gone.
if self.0.try_wait().expect("failed to wait for josh-proxy").is_some() {
return;
}
}
// If that didn't work (or we're not on Unix), kill it hard.
eprintln!(
"I have to kill josh-proxy the hard way, let's hope this does not break anything."
);
self.0.kill().expect("failed to SIGKILL josh-proxy");
}
}
Ok(Josh(josh))
}
pub fn exec(self) -> Result<()> {
// First, and crucially only once, run the auto-actions -- but not for all commands.
match &self {
@ -174,6 +227,8 @@ fn rustc_pull(commit: Option<String>) -> Result<()> {
if cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty().not() {
bail!("working directory must be clean before running `./miri rustc-pull`");
}
// Make sure josh is running.
let josh = Self::start_josh()?;
// Update rust-version file. As a separate commit, since making it part of
// the merge has confused the heck out of josh in the past.
@ -186,7 +241,7 @@ fn rustc_pull(commit: Option<String>) -> Result<()> {
.context("FAILED to commit rust-version file, something went wrong")?;
// Fetch given rustc commit.
cmd!(sh, "git fetch http://localhost:8000/rust-lang/rust.git@{commit}{JOSH_FILTER}.git")
cmd!(sh, "git fetch http://localhost:{JOSH_PORT}/rust-lang/rust.git@{commit}{JOSH_FILTER}.git")
.run()
.map_err(|e| {
// Try to un-do the previous `git commit`, to leave the repo in the state we found it it.
@ -202,6 +257,8 @@ fn rustc_pull(commit: Option<String>) -> Result<()> {
cmd!(sh, "git merge FETCH_HEAD --no-verify --no-ff -m {MERGE_COMMIT_MESSAGE}")
.run()
.context("FAILED to merge new commits, something went wrong")?;
drop(josh);
Ok(())
}
@ -213,6 +270,8 @@ fn rustc_push(github_user: String, branch: String) -> Result<()> {
if cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty().not() {
bail!("working directory must be clean before running `./miri rustc-push`");
}
// Make sure josh is running.
let josh = Self::start_josh()?;
// Find a repo we can do our preparation in.
if let Ok(rustc_git) = env::var("RUSTC_GIT") {
@ -249,6 +308,8 @@ fn rustc_push(github_user: String, branch: String) -> Result<()> {
}
cmd!(sh, "git fetch https://github.com/rust-lang/rust {base}").run()?;
cmd!(sh, "git push https://github.com/{github_user}/rust {base}:refs/heads/{branch}")
.ignore_stdout()
.ignore_stderr() // silence the "create GitHub PR" message
.run()?;
println!();
@ -257,7 +318,7 @@ fn rustc_push(github_user: String, branch: String) -> Result<()> {
println!("Pushing miri changes...");
cmd!(
sh,
"git push http://localhost:8000/{github_user}/rust.git{JOSH_FILTER}.git HEAD:{branch}"
"git push http://localhost:{JOSH_PORT}/{github_user}/rust.git{JOSH_FILTER}.git HEAD:{branch}"
)
.run()?;
println!();
@ -265,7 +326,7 @@ fn rustc_push(github_user: String, branch: String) -> Result<()> {
// Do a round-trip check to make sure the push worked as expected.
cmd!(
sh,
"git fetch http://localhost:8000/{github_user}/rust.git{JOSH_FILTER}.git {branch}"
"git fetch http://localhost:{JOSH_PORT}/{github_user}/rust.git{JOSH_FILTER}.git {branch}"
)
.ignore_stderr()
.read()?;
@ -278,6 +339,8 @@ fn rustc_push(github_user: String, branch: String) -> Result<()> {
"Confirmed that the push round-trips back to Miri properly. Please create a rustc PR:"
);
println!(" https://github.com/{github_user}/rust/pull/new/{branch}");
drop(josh);
Ok(())
}

2
src/tools/miri/miri-script/src/main.rs
View File

@ -1,3 +1,5 @@
#![allow(clippy::needless_question_mark)]
mod commands;
mod util;

1
src/tools/miri/src/lib.rs
View File

@ -32,6 +32,7 @@
clippy::needless_return,
clippy::bool_to_int_with_if,
clippy::box_default,
clippy::needless_question_mark,
// We are not implementing queries here so it's fine
rustc::potential_query_instability
)]