mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Auto merge of #17025 - lnicola:josh, r=lnicola

Browse Source 
internal: Use josh for subtree syncs
This commit is contained in:
bors 2024-04-21 16:39:18 +00:00
commit a200391f54
8 changed files with 251 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
src/tools/rust-analyzer/Cargo.lock generated
View File

@ -323,6 +323,27 @@ dependencies = [
"syn",
]
[[package]]
name = "directories"
version = "5.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35"
dependencies = [
"dirs-sys",
]
[[package]]
name = "dirs-sys"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c"
dependencies = [
"libc",
"option-ext",
"redox_users",
"windows-sys 0.48.0",
]
[[package]]
name = "dissimilar"
version = "1.0.7"
@ -898,6 +919,16 @@ dependencies = [
"libc",
]
[[package]]
name = "libredox"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
dependencies = [
"bitflags 2.4.2",
"libc",
]
[[package]]
name = "limit"
version = "0.0.0"
@ -1186,6 +1217,12 @@ version = "11.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
[[package]]
name = "option-ext"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
[[package]]
name = "parking_lot"
version = "0.12.1"
@ -1566,6 +1603,17 @@ dependencies = [
"bitflags 1.3.2",
]
[[package]]
name = "redox_users"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891"
dependencies = [
"getrandom",
"libredox",
"thiserror",
]
[[package]]
name = "rowan"
version = "0.15.15"
@ -2499,6 +2547,7 @@ name = "xtask"
version = "0.1.0"
dependencies = [
"anyhow",
"directories",
"flate2",
"itertools",
"proc-macro2",

1
src/tools/rust-analyzer/crates/rust-analyzer/tests/slow-tests/tidy.rs
View File

@ -144,6 +144,7 @@ MIT OR Apache-2.0
MIT OR Apache-2.0 OR Zlib
MIT OR Zlib OR Apache-2.0
MIT/Apache-2.0
MPL-2.0
Unlicense OR MIT
Unlicense/MIT
Zlib OR Apache-2.0 OR MIT

20
src/tools/rust-analyzer/docs/dev/README.md
View File

@ -229,12 +229,22 @@ Release steps:
* publishes the VS Code extension to the marketplace
* call the GitHub API for PR details
* create a new changelog in `rust-analyzer.github.io`
3. While the release is in progress, fill in the changelog
4. Commit & push the changelog
3. While the release is in progress, fill in the changelog.
4. Commit & push the changelog.
5. Run `cargo xtask publish-release-notes <CHANGELOG>` -- this will convert the changelog entry in AsciiDoc to Markdown and update the body of GitHub Releases entry.
6. Tweet
7. Inside `rust-analyzer`, run `cargo xtask promote` -- this will create a PR to rust-lang/rust updating rust-analyzer's subtree.
Self-approve the PR.
6. Tweet.
7. Make a new branch and run `cargo xtask rustc-pull`, open a PR, and merge it.
This will pull any changes from `rust-lang/rust` into `rust-analyzer`.
8. Switch to `master`, pull, then run `cargo xtask rustc-push --rust-path ../rust-rust-analyzer --rust-fork matklad/rust`.
Replace `matklad/rust` with your own fork of `rust-lang/rust`.
You can use the token to authenticate when you get prompted for a password, since `josh` will push over HTTPS, not SSH.
This will push the `rust-analyzer` changes to your fork.
You can then open a PR against `rust-lang/rust`.
Note: besides the `rust-rust-analyzer` clone, the Josh cache (stored under `~/.cache/rust-analyzer-josh`) will contain a bare clone of `rust-lang/rust`.
This currently takes about 3.5 GB.
This [HackMD](https://hackmd.io/7pOuxnkdQDaL1Y1FQr65xg) has details about how `josh` syncs work.
If the GitHub Actions release fails because of a transient problem like a timeout, you can re-run the job from the Actions console.
If it fails because of something that needs to be fixed, remove the release tag (if needed), fix the problem, then start over.

1
src/tools/rust-analyzer/rust-version Normal file
View File

@ -0,0 +1 @@
688c30dc9f8434d63bddb65bd6a4d2258d19717c

1
src/tools/rust-analyzer/xtask/Cargo.toml
View File

@ -8,6 +8,7 @@ rust-version.workspace = true
[dependencies]
anyhow.workspace = true
directories = "5.0"
flate2 = "1.0.24"
write-json = "0.1.2"
xshell.workspace = true

38
src/tools/rust-analyzer/xtask/src/flags.rs
View File

@ -14,16 +14,16 @@ xflags::xflags! {
cmd install {
/// Install only VS Code plugin.
optional --client
/// One of 'code', 'code-exploration', 'code-insiders', 'codium', or 'code-oss'.
/// One of `code`, `code-exploration`, `code-insiders`, `codium`, or `code-oss`.
optional --code-bin name: String
/// Install only the language server.
optional --server
/// Use mimalloc allocator for server
/// Use mimalloc allocator for server.
optional --mimalloc
/// Use jemalloc allocator for server
/// Use jemalloc allocator for server.
optional --jemalloc
/// build in release with debug info set to 2
/// build in release with debug info set to 2.
optional --dev-rel
}
@ -32,9 +32,21 @@ xflags::xflags! {
cmd release {
optional --dry-run
}
cmd promote {
optional --dry-run
cmd rustc-pull {
/// rustc commit to pull.
optional --commit refspec: String
}
cmd rustc-push {
/// rust local path, e.g. `../rust-rust-analyzer`.
required --rust-path rust_path: String
/// rust fork name, e.g. `matklad/rust`.
required --rust-fork rust_fork: String
/// branch name.
optional --branch branch: String
}
cmd dist {
/// Use mimalloc allocator for server
optional --mimalloc
@ -77,7 +89,8 @@ pub enum XtaskCmd {
Install(Install),
FuzzTests(FuzzTests),
Release(Release),
Promote(Promote),
RustcPull(RustcPull),
RustcPush(RustcPush),
Dist(Dist),
PublishReleaseNotes(PublishReleaseNotes),
Metrics(Metrics),
@ -104,8 +117,15 @@ pub struct Release {
}
#[derive(Debug)]
pub struct Promote {
pub dry_run: bool,
pub struct RustcPull {
pub commit: Option<String>,
}
#[derive(Debug)]
pub struct RustcPush {
pub rust_path: String,
pub rust_fork: String,
pub branch: Option<String>,
}
#[derive(Debug)]

3
src/tools/rust-analyzer/xtask/src/main.rs
View File

@ -34,7 +34,8 @@ fn main() -> anyhow::Result<()> {
flags::XtaskCmd::Install(cmd) => cmd.run(sh),
flags::XtaskCmd::FuzzTests(_) => run_fuzzer(sh),
flags::XtaskCmd::Release(cmd) => cmd.run(sh),
flags::XtaskCmd::Promote(cmd) => cmd.run(sh),
flags::XtaskCmd::RustcPull(cmd) => cmd.run(sh),
flags::XtaskCmd::RustcPush(cmd) => cmd.run(sh),
flags::XtaskCmd::Dist(cmd) => cmd.run(sh),
flags::XtaskCmd::PublishReleaseNotes(cmd) => cmd.run(sh),
flags::XtaskCmd::Metrics(cmd) => cmd.run(sh),

171
src/tools/rust-analyzer/xtask/src/release.rs
View File

@ -1,5 +1,12 @@
mod changelog;
use std::process::{Command, Stdio};
use std::thread;
use std::time::Duration;
use anyhow::{bail, Context as _};
use directories::ProjectDirs;
use stdx::JodChild;
use xshell::{cmd, Shell};
use crate::{codegen, date_iso, flags, is_release_tag, project_root};
@ -71,26 +78,154 @@ impl flags::Release {
}
}
impl flags::Promote {
// git sync implementation adapted from https://github.com/rust-lang/miri/blob/62039ac/miri-script/src/commands.rs
impl flags::RustcPull {
pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> {
let _dir = sh.push_dir("../rust-rust-analyzer");
cmd!(sh, "git switch master").run()?;
cmd!(sh, "git fetch upstream").run()?;
cmd!(sh, "git reset --hard upstream/master").run()?;
let date = date_iso(sh)?;
let branch = format!("rust-analyzer-{date}");
cmd!(sh, "git switch -c {branch}").run()?;
cmd!(sh, "git subtree pull -m ':arrow_up: rust-analyzer' -P src/tools/rust-analyzer rust-analyzer release").run()?;
if !self.dry_run {
cmd!(sh, "git push -u origin {branch}").run()?;
cmd!(
sh,
"xdg-open https://github.com/matklad/rust/pull/new/{branch}?body=r%3F%20%40ghost"
)
.run()?;
sh.change_dir(project_root());
let commit = self.commit.map(Result::Ok).unwrap_or_else(|| {
let rust_repo_head =
cmd!(sh, "git ls-remote https://github.com/rust-lang/rust/ HEAD").read()?;
rust_repo_head
.split_whitespace()
.next()
.map(|front| front.trim().to_owned())
.ok_or_else(|| anyhow::format_err!("Could not obtain Rust repo HEAD from remote."))
})?;
// Make sure the repo is clean.
if !cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty() {
bail!("working directory must be clean before running `cargo xtask pull`");
}
// Make sure josh is running.
let josh = start_josh()?;
// Update rust-version file. As a separate commit, since making it part of
// the merge has confused the heck out of josh in the past.
// We pass `--no-verify` to avoid running any git hooks that might exist,
// in case they dirty the repository.
sh.write_file("rust-version", format!("{commit}\n"))?;
const PREPARING_COMMIT_MESSAGE: &str = "Preparing for merge from rust-lang/rust";
cmd!(sh, "git commit rust-version --no-verify -m {PREPARING_COMMIT_MESSAGE}")
.run()
.context("FAILED to commit rust-version file, something went wrong")?;
// Fetch given rustc commit.
cmd!(sh, "git fetch http://localhost:{JOSH_PORT}/rust-lang/rust.git@{commit}{JOSH_FILTER}.git")
.run()
.map_err(|e| {
// Try to un-do the previous `git commit`, to leave the repo in the state we found it it.
cmd!(sh, "git reset --hard HEAD^")
.run()
.expect("FAILED to clean up again after failed `git fetch`, sorry for that");
e
})
.context("FAILED to fetch new commits, something went wrong (committing the rust-version file has been undone)")?;
// Merge the fetched commit.
const MERGE_COMMIT_MESSAGE: &str = "Merge from rust-lang/rust";
cmd!(sh, "git merge FETCH_HEAD --no-verify --no-ff -m {MERGE_COMMIT_MESSAGE}")
.run()
.context("FAILED to merge new commits, something went wrong")?;
drop(josh);
Ok(())
}
}
impl flags::RustcPush {
pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> {
let branch = self.branch.as_deref().unwrap_or("sync-from-ra");
let rust_path = self.rust_path;
let rust_fork = self.rust_fork;
sh.change_dir(project_root());
let base = sh.read_file("rust-version")?.trim().to_owned();
// Make sure the repo is clean.
if !cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty() {
bail!("working directory must be clean before running `cargo xtask push`");
}
// Make sure josh is running.
let josh = start_josh()?;
// Find a repo we can do our preparation in.
sh.change_dir(rust_path);
// Prepare the branch. Pushing works much better if we use as base exactly
// the commit that we pulled from last time, so we use the `rust-version`
// file to find out which commit that would be.
println!("Preparing {rust_fork} (base: {base})...");
if cmd!(sh, "git fetch https://github.com/{rust_fork} {branch}")
.ignore_stderr()
.read()
.is_ok()
{
bail!(
"The branch `{branch}` seems to already exist in `https://github.com/{rust_fork}`. Please delete it and try again."
);
}
cmd!(sh, "git fetch https://github.com/rust-lang/rust {base}").run()?;
cmd!(sh, "git push https://github.com/{rust_fork} {base}:refs/heads/{branch}")
.ignore_stdout()
.ignore_stderr() // silence the "create GitHub PR" message
.run()?;
println!();
// Do the actual push.
sh.change_dir(project_root());
println!("Pushing rust-analyzer changes...");
cmd!(
sh,
"git push http://localhost:{JOSH_PORT}/{rust_fork}.git{JOSH_FILTER}.git HEAD:{branch}"
)
.run()?;
println!();
// Do a round-trip check to make sure the push worked as expected.
cmd!(
sh,
"git fetch http://localhost:{JOSH_PORT}/{rust_fork}.git{JOSH_FILTER}.git {branch}"
)
.ignore_stderr()
.read()?;
let head = cmd!(sh, "git rev-parse HEAD").read()?;
let fetch_head = cmd!(sh, "git rev-parse FETCH_HEAD").read()?;
if head != fetch_head {
bail!("Josh created a non-roundtrip push! Do NOT merge this into rustc!");
}
println!("Confirmed that the push round-trips back to rust-analyzer properly. Please create a rustc PR:");
// https://github.com/github-linguist/linguist/compare/master...octocat:linguist:master
let fork_path = rust_fork.replace('/', ":");
println!(
" https://github.com/rust-lang/rust/compare/{fork_path}:{branch}?quick_pull=1&title=Subtree+update+of+rust-analyzer&body=r?+@ghost"
);
drop(josh);
Ok(())
}
}
/// Used for rustc syncs.
const JOSH_FILTER: &str =
":rev(55d9a533b309119c8acd13061581b43ae8840823:prefix=src/tools/rust-analyzer):/src/tools/rust-analyzer";
const JOSH_PORT: &str = "42042";
fn start_josh() -> anyhow::Result<impl Drop> {
// Determine cache directory.
let local_dir = {
let user_dirs = ProjectDirs::from("org", "rust-lang", "rust-analyzer-josh").unwrap();
user_dirs.cache_dir().to_owned()
};
// Start josh, silencing its output.
let mut cmd = Command::new("josh-proxy");
cmd.arg("--local").arg(local_dir);
cmd.arg("--remote").arg("https://github.com");
cmd.arg("--port").arg(JOSH_PORT);
cmd.arg("--no-background");
cmd.stdout(Stdio::null());
cmd.stderr(Stdio::null());
let josh = cmd.spawn().context("failed to start josh-proxy, make sure it is installed")?;
// Give it some time so hopefully the port is open. (100ms was not enough.)
thread::sleep(Duration::from_millis(200));
Ok(JodChild(josh))
}