Skip to content

Commit

Permalink
Implement sync subcommand in clippy_dev
Browse files Browse the repository at this point in the history
Now that JOSH is used to sync, it is much easier to script the sync process.
This introduces the two commands `sync pull` and `sync push`. The first one will
pull changes from the Rust repo, the second one will push the changes to the
Rust repo. For details, see the documentation in the book.
  • Loading branch information
flip1995 committed Jun 28, 2024
1 parent 7406be6 commit 1c08298
Show file tree
Hide file tree
Showing 4 changed files with 283 additions and 1 deletion.
3 changes: 3 additions & 0 deletions clippy_dev/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,15 @@ edition = "2021"

[dependencies]
aho-corasick = "1.0"
chrono = { version = "0.4.38", default-features = false, features = ["clock"] }
clap = { version = "4.4", features = ["derive"] }
directories = "5"
indoc = "1.0"
itertools = "0.12"
opener = "0.6"
shell-escape = "0.1"
walkdir = "2.3"
xshell = "0.2"

[features]
deny-warnings = []
Expand Down
1 change: 1 addition & 0 deletions clippy_dev/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,5 +21,6 @@ pub mod lint;
pub mod new_lint;
pub mod serve;
pub mod setup;
pub mod sync;
pub mod update_lints;
pub mod utils;
41 changes: 40 additions & 1 deletion clippy_dev/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
#![warn(rust_2018_idioms, unused_lifetimes)]

use clap::{Args, Parser, Subcommand};
use clippy_dev::{dogfood, fmt, lint, new_lint, serve, setup, update_lints, utils};
use clippy_dev::{dogfood, fmt, lint, new_lint, serve, setup, sync, update_lints, utils};
use std::convert::Infallible;

fn main() {
Expand Down Expand Up @@ -75,6 +75,15 @@ fn main() {
uplift,
} => update_lints::rename(&old_name, new_name.as_ref().unwrap_or(&old_name), uplift),
DevCommand::Deprecate { name, reason } => update_lints::deprecate(&name, reason.as_deref()),
DevCommand::Sync(SyncCommand { subcommand }) => match subcommand {
SyncSubcommand::Pull => sync::rustc_pull(),
SyncSubcommand::Push {
repo_path,
user,
branch,
force,
} => sync::rustc_push(repo_path, &user, &branch, force),
},
}
}

Expand Down Expand Up @@ -225,6 +234,8 @@ enum DevCommand {
/// The reason for deprecation
reason: Option<String>,
},
/// Sync between the rust repo and the Clippy repo
Sync(SyncCommand),
}

#[derive(Args)]
Expand Down Expand Up @@ -291,3 +302,31 @@ enum RemoveSubcommand {
/// Remove the tasks added with 'cargo dev setup vscode-tasks'
VscodeTasks,
}

#[derive(Args)]
struct SyncCommand {
#[command(subcommand)]
subcommand: SyncSubcommand,
}

#[derive(Subcommand)]
enum SyncSubcommand {
/// Pull changes from rustc and update the toolchain
Pull,
/// Push changes to rustc
Push {
/// The path to a rustc repo that will be used for pushing changes
repo_path: String,
#[arg(long)]
/// The GitHub username to use for pushing changes
user: String,
#[arg(long, short, default_value = "clippy-subtree-update")]
/// The branch to push to
///
/// This is mostly for experimentation and usually the default should be used.
branch: String,
#[arg(long, short)]
/// Force push changes
force: bool,
},
}
239 changes: 239 additions & 0 deletions clippy_dev/src/sync.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,239 @@
use std::fmt::Write;
use std::path::Path;
use std::process;
use std::process::exit;

use chrono::offset::Utc;
use xshell::{cmd, Shell};

use crate::utils::{clippy_project_root, replace_region_in_file, UpdateMode};

const JOSH_FILTER: &str = ":rev(2efebd2f0c03dabbe5c3ad7b4ebfbd99238d1fb2:prefix=src/tools/clippy):/src/tools/clippy";
const JOSH_PORT: &str = "42042";

fn start_josh() -> impl Drop {
// Create a wrapper that stops it on drop.
struct Josh(process::Child);
impl Drop for Josh {
fn drop(&mut self) {
#[cfg(unix)]
{
// Try to gracefully shut it down.
process::Command::new("kill")
.args(["-s", "INT", &self.0.id().to_string()])
.output()
.expect("failed to SIGINT josh-proxy");
// Sadly there is no "wait with timeout"... so we just give it some time to finish.
std::thread::sleep(std::time::Duration::from_secs(1));
// Now hopefully it is gone.
if self.0.try_wait().expect("failed to wait for josh-proxy").is_some() {
return;
}
}
// If that didn't work (or we're not on Unix), kill it hard.
eprintln!("I have to kill josh-proxy the hard way, let's hope this does not break anything.");
self.0.kill().expect("failed to SIGKILL josh-proxy");
}
}

// Determine cache directory.
let local_dir = {
let user_dirs = directories::ProjectDirs::from("org", "rust-lang", "clippy-josh").unwrap();
user_dirs.cache_dir().to_owned()
};
println!("Using local cache directory: {}", local_dir.display());

// Start josh, silencing its output.
let mut cmd = process::Command::new("josh-proxy");
cmd.arg("--local").arg(local_dir);
cmd.arg("--remote").arg("https://github.com");
cmd.arg("--port").arg(JOSH_PORT);
cmd.arg("--no-background");
cmd.stdout(process::Stdio::null());
cmd.stderr(process::Stdio::null());
let josh = cmd
.spawn()
.expect("failed to start josh-proxy, make sure it is installed");
// Give it some time so hopefully the port is open.
std::thread::sleep(std::time::Duration::from_secs(1));

Josh(josh)
}

fn rustc_hash() -> String {
let sh = Shell::new().expect("failed to create shell");
// Make sure we pick up the updated toolchain (usually rustup pins the toolchain
// inside a single cargo/rustc invocation via this env var).
sh.set_var("RUSTUP_TOOLCHAIN", "");
cmd!(sh, "rustc --version --verbose")
.read()
.expect("failed to run `rustc -vV`")
.lines()
.find(|line| line.starts_with("commit-hash:"))
.expect("failed to parse `rustc -vV`")
.split_whitespace()
.last()
.expect("failed to get commit from `rustc -vV`")
.to_string()
}

fn assert_clean_repo(sh: &Shell) {
if !cmd!(sh, "git status --untracked-files=no --porcelain")
.read()
.expect("failed to run git status")
.is_empty()
{
eprintln!("working directory must be clean before running `cargo dev sync pull`");
exit(1);
}
}

pub fn rustc_pull() {
const MERGE_COMMIT_MESSAGE: &str = "Merge from rustc";

let sh = Shell::new().expect("failed to create shell");
sh.change_dir(clippy_project_root());

assert_clean_repo(&sh);

// Update rust-toolchain file
let date = Utc::now().format("%Y-%m-%d").to_string();
replace_region_in_file(
UpdateMode::Change,
Path::new("rust-toolchain"),
"# begin autogenerated version\n",
"# end autogenerated version",
|res| {
writeln!(res, "channel = \"nightly-{date}\"").unwrap();
},
);

let message = format!("Bump nightly version -> {date}");
cmd!(sh, "git commit rust-toolchain --no-verify -m {message}")
.run()
.expect("FAILED to commit rust-toolchain file, something went wrong");

let commit = rustc_hash();

// Make sure josh is running in this scope
{
let _josh = start_josh();

// Fetch given rustc commit.
cmd!(
sh,
"git fetch http://localhost:{JOSH_PORT}/rust-lang/rust.git@{commit}{JOSH_FILTER}.git"
)
.run()
.inspect_err(|_| {
// Try to un-do the previous `git commit`, to leave the repo in the state we found it.
cmd!(sh, "git reset --hard HEAD^")
.run()
.expect("FAILED to clean up again after failed `git fetch`, sorry for that");
})
.expect("FAILED to fetch new commits, something went wrong");
}

// This should not add any new root commits. So count those before and after merging.
let num_roots = || -> u32 {
cmd!(sh, "git rev-list HEAD --max-parents=0 --count")
.read()
.expect("failed to determine the number of root commits")
.parse::<u32>()
.unwrap()
};
let num_roots_before = num_roots();

// Merge the fetched commit.
cmd!(sh, "git merge FETCH_HEAD --no-verify --no-ff -m {MERGE_COMMIT_MESSAGE}")
.run()
.expect("FAILED to merge new commits, something went wrong");

// Check that the number of roots did not increase.
if num_roots() != num_roots_before {
eprintln!("Josh created a new root commit. This is probably not the history you want.");
exit(1);
}
}

pub(crate) const PUSH_PR_DESCRIPTION: &str = "Sync from Clippy commit:";

pub fn rustc_push(rustc_path: String, github_user: &str, branch: &str, force: bool) {
let sh = Shell::new().expect("failed to create shell");
sh.change_dir(clippy_project_root());

assert_clean_repo(&sh);

// Prepare the branch. Pushing works much better if we use as base exactly
// the commit that we pulled from last time, so we use the `rustc --version`
// to find out which commit that would be.
let base = rustc_hash();

println!("Preparing {github_user}/rust (base: {base})...");
sh.change_dir(rustc_path);
if !force
&& cmd!(sh, "git fetch https://github.com/{github_user}/rust {branch}")
.ignore_stderr()
.read()
.is_ok()
{
eprintln!(
"The branch '{branch}' seems to already exist in 'https://github.com/{github_user}/rust'. Please delete it and try again."
);
exit(1);
}
cmd!(sh, "git fetch https://github.com/rust-lang/rust {base}")
.run()
.expect("failed to fetch base commit");
let force_flag = if force { "--force" } else { "" };
cmd!(
sh,
"git push https://github.com/{github_user}/rust {base}:refs/heads/{branch} {force_flag}"
)
.ignore_stdout()
.ignore_stderr() // silence the "create GitHub PR" message
.run()
.expect("failed to push base commit to the new branch");

// Make sure josh is running in this scope
{
let _josh = start_josh();

// Do the actual push.
sh.change_dir(clippy_project_root());
println!("Pushing Clippy changes...");
cmd!(
sh,
"git push http://localhost:{JOSH_PORT}/{github_user}/rust.git{JOSH_FILTER}.git HEAD:{branch}"
)
.run()
.expect("failed to push changes to Josh");

// Do a round-trip check to make sure the push worked as expected.
cmd!(
sh,
"git fetch http://localhost:{JOSH_PORT}/{github_user}/rust.git{JOSH_FILTER}.git {branch}"
)
.ignore_stderr()
.read()
.expect("failed to fetch the branch from Josh");
}

let head = cmd!(sh, "git rev-parse HEAD")
.read()
.expect("failed to get HEAD commit");
let fetch_head = cmd!(sh, "git rev-parse FETCH_HEAD")
.read()
.expect("failed to get FETCH_HEAD");
if head != fetch_head {
eprintln!("Josh created a non-roundtrip push! Do NOT merge this into rustc!");
exit(1);
}
println!("Confirmed that the push round-trips back to Clippy properly. Please create a rustc PR:");
let description = format!("{}+rust-lang/rust-clippy@{head}", PUSH_PR_DESCRIPTION.replace(' ', "+"));
println!(
// Open PR with `subtree update` title to silence the `no-merges` triagebot check
// See https://github.com/rust-lang/rust/pull/114157
" https://github.com/rust-lang/rust/compare/{github_user}:{branch}?quick_pull=1&title=Clippy+subtree+update&body=r?+@ghost%0A%0A{description}"
);
}

0 comments on commit 1c08298

Please sign in to comment.