Use josh for subtree syncs

Author: lnicola

Cargo.lock

@@ -144,6 +144,7 @@ MIT OR Apache-2.0
 MIT OR Apache-2.0 OR Zlib
 MIT OR Zlib OR Apache-2.0
 MIT/Apache-2.0
+MPL-2.0
 Unlicense OR MIT
 Unlicense/MIT
 Zlib OR Apache-2.0 OR MIT

crates/rust-analyzer/tests/slow-tests/tidy.rs

@@ -0,0 +1 @@
+688c30dc9f8434d63bddb65bd6a4d2258d19717c

rust-version

@@ -8,6 +8,7 @@ rust-version.workspace = true
 
 [dependencies]
 anyhow.workspace = true
+directories = "5.0"
 flate2 = "1.0.24"
 write-json = "0.1.2"
 xshell.workspace = true

xtask/Cargo.toml

@@ -32,9 +32,12 @@ xflags::xflags! {
         cmd release {
             optional --dry-run
         }
-        cmd promote {
-            optional --dry-run
+
+        cmd pull {
+            optional --commit refspec: String
         }
+        cmd push {}
+
         cmd dist {
             /// Use mimalloc allocator for server
             optional --mimalloc
@@ -77,7 +80,8 @@ pub enum XtaskCmd {
     Install(Install),
     FuzzTests(FuzzTests),
     Release(Release),
-    Promote(Promote),
+    Pull(Pull),
+    Push(Push),
     Dist(Dist),
     PublishReleaseNotes(PublishReleaseNotes),
     Metrics(Metrics),
@@ -104,10 +108,13 @@ pub struct Release {
 }
 
 #[derive(Debug)]
-pub struct Promote {
-    pub dry_run: bool,
+pub struct Pull {
+    pub commit: Option<String>,
 }
 
+#[derive(Debug)]
+pub struct Push;
+
 #[derive(Debug)]
 pub struct Dist {
     pub mimalloc: bool,

xtask/src/flags.rs

@@ -34,7 +34,8 @@ fn main() -> anyhow::Result<()> {
         flags::XtaskCmd::Install(cmd) => cmd.run(sh),
         flags::XtaskCmd::FuzzTests(_) => run_fuzzer(sh),
         flags::XtaskCmd::Release(cmd) => cmd.run(sh),
-        flags::XtaskCmd::Promote(cmd) => cmd.run(sh),
+        flags::XtaskCmd::Pull(cmd) => cmd.run(sh),
+        flags::XtaskCmd::Push(cmd) => cmd.run(sh),
         flags::XtaskCmd::Dist(cmd) => cmd.run(sh),
         flags::XtaskCmd::PublishReleaseNotes(cmd) => cmd.run(sh),
         flags::XtaskCmd::Metrics(cmd) => cmd.run(sh),

xtask/src/main.rs

@@ -1,5 +1,12 @@
 mod changelog;
 
+use std::process::{Command, Stdio};
+use std::time::Duration;
+use std::{env, thread};
+
+use anyhow::{bail, Context as _};
+use directories::ProjectDirs;
+use stdx::JodChild;
 use xshell::{cmd, Shell};
 
 use crate::{codegen, date_iso, flags, is_release_tag, project_root};
@@ -71,26 +78,157 @@ impl flags::Release {
     }
 }
 
-impl flags::Promote {
+// git sync implementation adapted from https://github.com/rust-lang/miri/blob/62039ac/miri-script/src/commands.rs
+impl flags::Pull {
     pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> {
-        let _dir = sh.push_dir("../rust-rust-analyzer");
-        cmd!(sh, "git switch master").run()?;
-        cmd!(sh, "git fetch upstream").run()?;
-        cmd!(sh, "git reset --hard upstream/master").run()?;
+        sh.change_dir(project_root());
+        let commit = self.commit.map(Result::Ok).unwrap_or_else(|| {
+            let rust_repo_head =
+                cmd!(sh, "git ls-remote https://github.com/rust-lang/rust/ HEAD").read()?;
+            rust_repo_head
+                .split_whitespace()
+                .next()
+                .map(|front| front.trim().to_owned())
+                .ok_or_else(|| anyhow::format_err!("Could not obtain Rust repo HEAD from remote."))
+        })?;
+        // Make sure the repo is clean.
+        if !cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty() {
+            bail!("working directory must be clean before running `cargo xtask pull`");
+        }
+        // Make sure josh is running.
+        let josh = start_josh()?;
 
-        let date = date_iso(sh)?;
-        let branch = format!("rust-analyzer-{date}");
-        cmd!(sh, "git switch -c {branch}").run()?;
-        cmd!(sh, "git subtree pull -m ':arrow_up: rust-analyzer' -P src/tools/rust-analyzer rust-analyzer release").run()?;
+        // Update rust-version file. As a separate commit, since making it part of
+        // the merge has confused the heck out of josh in the past.
+        // We pass `--no-verify` to avoid running any git hooks that might exist,
+        // in case they dirty the repository.
+        sh.write_file("rust-version", format!("{commit}\n"))?;
+        const PREPARING_COMMIT_MESSAGE: &str = "Preparing for merge from rustc";
+        cmd!(sh, "git commit rust-version --no-verify -m {PREPARING_COMMIT_MESSAGE}")
+            .run()
+            .context("FAILED to commit rust-version file, something went wrong")?;
 
-        if !self.dry_run {
-            cmd!(sh, "git push -u origin {branch}").run()?;
-            cmd!(
-                sh,
-                "xdg-open https://github.com/matklad/rust/pull/new/{branch}?body=r%3F%20%40ghost"
-            )
+        // Fetch given rustc commit.
+        cmd!(sh, "git fetch http://localhost:{JOSH_PORT}/rust-lang/rust.git@{commit}{JOSH_FILTER}.git")
+            .run()
+            .map_err(|e| {
+                // Try to un-do the previous `git commit`, to leave the repo in the state we found it it.
+                cmd!(sh, "git reset --hard HEAD^")
+                    .run()
+                    .expect("FAILED to clean up again after failed `git fetch`, sorry for that");
+                e
+            })
+            .context("FAILED to fetch new commits, something went wrong (committing the rust-version file has been undone)")?;
+
+        // Merge the fetched commit.
+        const MERGE_COMMIT_MESSAGE: &str = "Merge from downstream";
+        cmd!(sh, "git merge FETCH_HEAD --no-verify --no-ff -m {MERGE_COMMIT_MESSAGE}")
+            .run()
+            .context("FAILED to merge new commits, something went wrong")?;
+
+        drop(josh);
+        Ok(())
+    }
+}
+
+impl flags::Push {
+    pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> {
+        let branch = "sync-from-ra";
+        let Ok(github_user) = env::var("GITHUB_USER") else {
+            bail!("please set `GITHUB_USER` to the  GitHub username");
+        };
+
+        sh.change_dir(project_root());
+        let base = sh.read_file("rust-version")?.trim().to_owned();
+        // Make sure the repo is clean.
+        if !cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty() {
+            bail!("working directory must be clean before running `cargo xtask push`");
+        }
+        // Make sure josh is running.
+        let josh = start_josh()?;
+
+        // Find a repo we can do our preparation in.
+        if let Ok(rustc_git) = env::var("RUSTC_GIT") {
+            // If rustc_git is `Some`, we'll use an existing fork for the branch updates.
+            sh.change_dir(rustc_git);
+        } else {
+            bail!("please set `RUSTC_GIT` to a `rust-lang/rust` clone");
+        };
+        // Prepare the branch. Pushing works much better if we use as base exactly
+        // the commit that we pulled from last time, so we use the `rust-version`
+        // file to find out which commit that would be.
+        println!("Preparing {github_user}/rust (base: {base})...");
+        if cmd!(sh, "git fetch https://github.com/{github_user}/rust {branch}")
+            .ignore_stderr()
+            .read()
+            .is_ok()
+        {
+            bail!(
+                "The branch '{branch}' seems to already exist in 'https://github.com/{github_user}/rust'. Please delete it and try again."
+            );
+        }
+        cmd!(sh, "git fetch https://github.com/rust-lang/rust {base}").run()?;
+        cmd!(sh, "git push https://github.com/{github_user}/rust {base}:refs/heads/{branch}")
+            .ignore_stdout()
+            .ignore_stderr() // silence the "create GitHub PR" message
             .run()?;
+        println!();
+
+        // Do the actual push.
+        sh.change_dir(project_root());
+        println!("Pushing rust-analyzer changes...");
+        cmd!(
+            sh,
+            "git push http://localhost:{JOSH_PORT}/{github_user}/rust.git{JOSH_FILTER}.git HEAD:{branch}"
+        )
+        .run()?;
+        println!();
+
+        // Do a round-trip check to make sure the push worked as expected.
+        cmd!(
+            sh,
+            "git fetch http://localhost:{JOSH_PORT}/{github_user}/rust.git{JOSH_FILTER}.git {branch}"
+        )
+        .ignore_stderr()
+        .read()?;
+        let head = cmd!(sh, "git rev-parse HEAD").read()?;
+        let fetch_head = cmd!(sh, "git rev-parse FETCH_HEAD").read()?;
+        if head != fetch_head {
+            bail!("Josh created a non-roundtrip push! Do NOT merge this into rustc!");
         }
+        println!("Confirmed that the push round-trips back to rust-analyzer properly. Please create a rustc PR:");
+        println!(
+            "    https://github.com/rust-lang/rust/compare/{github_user}:{branch}?quick_pull=1&title=Subtree+update+of+rust-analyzer&body=r?+@ghost"
+        );
+
+        drop(josh);
         Ok(())
     }
 }
+
+/// Used for rustc syncs.
+const JOSH_FILTER: &str =
+    ":rev(f5a9250147f6569d8d89334dc9cca79c0322729f:prefix=src/tools/rust-analyzer):/src/tools/rust-analyzer";
+const JOSH_PORT: &str = "42042";
+
+fn start_josh() -> anyhow::Result<impl Drop> {
+    // Determine cache directory.
+    let local_dir = {
+        let user_dirs = ProjectDirs::from("org", "rust-lang", "rust-analyzer-josh").unwrap();
+        user_dirs.cache_dir().to_owned()
+    };
+
+    // Start josh, silencing its output.
+    let mut cmd = Command::new("josh-proxy");
+    cmd.arg("--local").arg(local_dir);
+    cmd.arg("--remote").arg("https://github.com");
+    cmd.arg("--port").arg(JOSH_PORT);
+    cmd.arg("--no-background");
+    cmd.stdout(Stdio::null());
+    cmd.stderr(Stdio::null());
+    let josh = cmd.spawn().context("failed to start josh-proxy, make sure it is installed")?;
+    // Give it some time so hopefully the port is open. (100ms was not enough.)
+    thread::sleep(Duration::from_millis(200));
+
+    Ok(JodChild(josh))
+}