From 4950b090b61823cce0872d63795cb1152697a7f0 Mon Sep 17 00:00:00 2001 From: Essien Ita Essien <34972+essiene@users.noreply.github.com> Date: Sat, 23 Nov 2024 21:04:47 +0000 Subject: [PATCH] jj-lib: refactor `jj git clone|fetch` and `to use new GitFetch api directly. * Make the new `GitFetch` api public. * Rewrite all tests that used `git::fetch*` to use the new API. One interesting artifact of the new API is that it holds onto a reference of the `tx.repo_mut()`, so the GitFetch object needs to fall go of scope completely before we can make any other mutable calls to `tx`. * Delete the `git::fetch` * Update `jj git clone` and `git_fetch` in `cli/src/git_utils.rs` to use the new api directly. Removing one redundant layer of indirection. * This fixes #4923 as it first fetches from all remotes before `import_refs()` is called, so there is no race condition if the same commit is treated differently in different remotes specified in the same command. Fixes: #4923 --- cli/src/commands/git/clone.rs | 44 +++--- cli/src/git_util.rs | 61 ++++---- cli/tests/test_git_fetch.rs | 10 +- lib/src/git.rs | 33 +--- lib/tests/test_git.rs | 287 ++++++++++++++++++++-------------- 5 files changed, 235 insertions(+), 200 deletions(-) diff --git a/cli/src/commands/git/clone.rs b/cli/src/commands/git/clone.rs index 57f0261089..d240343b2f 100644 --- a/cli/src/commands/git/clone.rs +++ b/cli/src/commands/git/clone.rs @@ -21,6 +21,7 @@ use std::path::PathBuf; use jj_lib::config::ConfigNamePathBuf; use jj_lib::git; +use jj_lib::git::GitFetch; use jj_lib::git::GitFetchError; use jj_lib::git::GitFetchStats; use jj_lib::repo::Repo; @@ -221,27 +222,34 @@ fn do_git_clone( git_repo.remote(remote_name, source).unwrap(); let mut fetch_tx = workspace_command.start_transaction(); - let stats = with_remote_git_callbacks(ui, None, |cb| { - git::fetch( + let stats = with_remote_git_callbacks(ui, None, |cb| -> Result { + let git_settings = command.settings().git_settings(); + let mut git_fetch = GitFetch::new( fetch_tx.repo_mut(), &git_repo, - remote_name, - &[StringPattern::everything()], - cb, - &command.settings().git_settings(), - depth, - ) - }) - .map_err(|err| match err { - GitFetchError::NoSuchRemote(_) => { - panic!("shouldn't happen as we just created the git remote") - } - GitFetchError::GitImportError(err) => CommandError::from(err), - GitFetchError::InternalGitError(err) => map_git_error(err), - GitFetchError::InvalidBranchPattern => { - unreachable!("we didn't provide any globs") - } + &git_settings, + git::fetch_options(cb, depth), + ); + + let default_branch = git_fetch + .fetch(&[StringPattern::everything()], remote_name) + .map_err(|err| match err { + GitFetchError::NoSuchRemote(_) => { + panic!("shouldn't happen as we just created the git remote") + } + GitFetchError::GitImportError(err) => CommandError::from(err), + GitFetchError::InternalGitError(err) => map_git_error(err), + GitFetchError::InvalidBranchPattern => { + unreachable!("we didn't provide any globs") + } + })?; + let import_stats = git_fetch.import_refs()?; + Ok(GitFetchStats { + default_branch, + import_stats, + }) })?; + print_git_import_stats(ui, fetch_tx.repo(), &stats.import_stats, true)?; fetch_tx.finish(ui, "fetch from git remote into empty repo")?; Ok((workspace_command, stats)) diff --git a/cli/src/git_util.rs b/cli/src/git_util.rs index 1edb7a6216..e4d68feb0f 100644 --- a/cli/src/git_util.rs +++ b/cli/src/git_util.rs @@ -27,6 +27,7 @@ use itertools::Itertools; use jj_lib::git; use jj_lib::git::FailedRefExport; use jj_lib::git::FailedRefExportReason; +use jj_lib::git::GitFetch; use jj_lib::git::GitFetchError; use jj_lib::git::GitImportStats; use jj_lib::git::RefName; @@ -456,6 +457,9 @@ export or their "parent" bookmarks."#, Ok(()) } +// TODO: Move this back to cli/src/commands/git/fetch.rs +// With the new `GitFetch` api, this function is too specialized +// to the `jj git fetch` command and should not be reused. pub fn git_fetch( ui: &mut Ui, tx: &mut WorkspaceCommandTransaction, @@ -463,40 +467,39 @@ pub fn git_fetch( remotes: &[String], branch: &[StringPattern], ) -> Result<(), CommandError> { - let git_settings = tx.settings().git_settings(); - - for remote in remotes { - let stats = with_remote_git_callbacks(ui, None, |cb| { - git::fetch( + let import_stats = + with_remote_git_callbacks(ui, None, |cb| -> Result { + let git_settings = tx.settings().git_settings(); + let mut git_fetch = GitFetch::new( tx.repo_mut(), git_repo, - remote, - branch, - cb, &git_settings, - None, - ) - }) - .map_err(|err| match err { - GitFetchError::InvalidBranchPattern => { - if branch - .iter() - .any(|pattern| pattern.as_exact().is_some_and(|s| s.contains('*'))) - { - user_error_with_hint( - "Branch names may not include `*`.", - "Prefix the pattern with `glob:` to expand `*` as a glob", - ) - } else { - user_error(err) - } + git::fetch_options(cb, None), + ); + + for remote in remotes { + git_fetch.fetch(branch, remote).map_err(|err| match err { + GitFetchError::InvalidBranchPattern => { + if branch + .iter() + .any(|pattern| pattern.as_exact().is_some_and(|s| s.contains('*'))) + { + user_error_with_hint( + "Branch names may not include `*`.", + "Prefix the pattern with `glob:` to expand `*` as a glob", + ) + } else { + user_error(err) + } + } + GitFetchError::GitImportError(err) => err.into(), + GitFetchError::InternalGitError(err) => map_git_error(err), + _ => user_error(err), + })?; } - GitFetchError::GitImportError(err) => err.into(), - GitFetchError::InternalGitError(err) => map_git_error(err), - _ => user_error(err), + Ok(git_fetch.import_refs()?) })?; - print_git_import_stats(ui, tx.repo(), &stats.import_stats, true)?; - } + print_git_import_stats(ui, tx.repo(), &import_stats, true)?; warn_if_branches_not_found( ui, tx, diff --git a/cli/tests/test_git_fetch.rs b/cli/tests/test_git_fetch.rs index 232ddae660..e508c64ccc 100644 --- a/cli/tests/test_git_fetch.rs +++ b/cli/tests/test_git_fetch.rs @@ -237,10 +237,7 @@ fn test_git_fetch_nonexistent_remote() { &repo_path, &["git", "fetch", "--remote", "rem1", "--remote", "rem2"], ); - insta::assert_snapshot!(stderr, @r###" - bookmark: rem1@rem1 [new] untracked - Error: No git remote named 'rem2' - "###); + insta::assert_snapshot!(stderr, @"Error: No git remote named 'rem2'"); // No remote should have been fetched as part of the failing transaction insta::assert_snapshot!(get_bookmark_output(&test_env, &repo_path), @""); } @@ -254,10 +251,7 @@ fn test_git_fetch_nonexistent_remote_from_config() { test_env.add_config(r#"git.fetch = ["rem1", "rem2"]"#); let stderr = &test_env.jj_cmd_failure(&repo_path, &["git", "fetch"]); - insta::assert_snapshot!(stderr, @r###" - bookmark: rem1@rem1 [new] untracked - Error: No git remote named 'rem2' - "###); + insta::assert_snapshot!(stderr, @"Error: No git remote named 'rem2'"); // No remote should have been fetched as part of the failing transaction insta::assert_snapshot!(get_bookmark_output(&test_env, &repo_path), @""); } diff --git a/lib/src/git.rs b/lib/src/git.rs index cce00162b6..d0488f8e08 100644 --- a/lib/src/git.rs +++ b/lib/src/git.rs @@ -1229,7 +1229,7 @@ pub enum GitFetchError { InternalGitError(#[from] git2::Error), } -fn fetch_options( +pub fn fetch_options( callbacks: RemoteCallbacks<'_>, depth: Option, ) -> git2::FetchOptions<'_> { @@ -1251,7 +1251,7 @@ struct FetchedBranches { remote: String, } -struct GitFetch<'a> { +pub struct GitFetch<'a> { mut_repo: &'a mut MutableRepo, git_repo: &'a git2::Repository, git_settings: &'a GitSettings, @@ -1260,7 +1260,7 @@ struct GitFetch<'a> { } impl<'a> GitFetch<'a> { - fn new( + pub fn new( mut_repo: &'a mut MutableRepo, git_repo: &'a git2::Repository, git_settings: &'a GitSettings, @@ -1280,7 +1280,7 @@ impl<'a> GitFetch<'a> { /// /// Keeps track of the {branch_names, remote_name} pair the refs can be /// subsequently imported into the `jj` repo by calling `import_refs()`. - fn fetch( + pub fn fetch( &mut self, branch_names: &[StringPattern], remote_name: &str, @@ -1394,31 +1394,6 @@ pub struct GitFetchStats { pub import_stats: GitImportStats, } -#[tracing::instrument(skip(mut_repo, git_repo, callbacks))] -pub fn fetch( - mut_repo: &mut MutableRepo, - git_repo: &git2::Repository, - remote_name: &str, - branch_names: &[StringPattern], - callbacks: RemoteCallbacks<'_>, - git_settings: &GitSettings, - depth: Option, -) -> Result { - let mut git_fetch = GitFetch::new( - mut_repo, - git_repo, - git_settings, - fetch_options(callbacks, depth), - ); - let default_branch = git_fetch.fetch(branch_names, remote_name)?; - let import_stats = git_fetch.import_refs()?; - let stats = GitFetchStats { - default_branch, - import_stats, - }; - Ok(stats) -} - #[derive(Error, Debug, PartialEq)] pub enum GitPushError { #[error("No git remote named '{0}'")] diff --git a/lib/tests/test_git.rs b/lib/tests/test_git.rs index 7f773509f3..c7cbad1494 100644 --- a/lib/tests/test_git.rs +++ b/lib/tests/test_git.rs @@ -38,6 +38,7 @@ use jj_lib::commit_builder::CommitBuilder; use jj_lib::git; use jj_lib::git::FailedRefExportReason; use jj_lib::git::GitBranchPushTargets; +use jj_lib::git::GitFetch; use jj_lib::git::GitFetchError; use jj_lib::git::GitImportError; use jj_lib::git::GitPushError; @@ -2262,19 +2263,24 @@ fn test_fetch_empty_repo() { let git_settings = GitSettings::default(); let mut tx = test_data.repo.start_transaction(&test_data.settings); - let stats = git::fetch( - tx.repo_mut(), - &test_data.git_repo, - "origin", - &[StringPattern::everything()], - git::RemoteCallbacks::default(), - &git_settings, - None, - ) - .unwrap(); - // No default bookmark and no refs - assert_eq!(stats.default_branch, None); - assert!(stats.import_stats.abandoned_commits.is_empty()); + { + let mut git_fetch = GitFetch::new( + tx.repo_mut(), + &test_data.git_repo, + &git_settings, + git::fetch_options(git::RemoteCallbacks::default(), None), + ); + + let default_branch = git_fetch + .fetch(&[StringPattern::everything()], "origin") + .unwrap(); + let import_stats = git_fetch.import_refs().unwrap(); + + // No default bookmark and no refs + assert_eq!(default_branch, None); + assert!(import_stats.abandoned_commits.is_empty()); + }; + assert_eq!(*tx.repo_mut().view().git_refs(), btreemap! {}); assert_eq!(tx.repo_mut().view().bookmarks().count(), 0); } @@ -2289,21 +2295,26 @@ fn test_fetch_initial_commit_head_is_not_set() { let initial_git_commit = empty_git_commit(&test_data.origin_repo, "refs/heads/main", &[]); let mut tx = test_data.repo.start_transaction(&test_data.settings); - let stats = git::fetch( - tx.repo_mut(), - &test_data.git_repo, - "origin", - &[StringPattern::everything()], - git::RemoteCallbacks::default(), - &git_settings, - None, - ) - .unwrap(); - // No default bookmark because the origin repo's HEAD wasn't set - assert_eq!(stats.default_branch, None); - assert!(stats.import_stats.abandoned_commits.is_empty()); + { + let mut git_fetch = GitFetch::new( + tx.repo_mut(), + &test_data.git_repo, + &git_settings, + git::fetch_options(git::RemoteCallbacks::default(), None), + ); + + let default_branch = git_fetch + .fetch(&[StringPattern::everything()], "origin") + .unwrap(); + let import_stats = git_fetch.import_refs().unwrap(); + + // No default bookmark because the origin repo's HEAD wasn't set + assert_eq!(default_branch, None); + assert!(import_stats.abandoned_commits.is_empty()); + } + let repo = tx.commit("test").unwrap(); - // The initial commit is visible after git::fetch(). + // The initial commit is visible after git_fetch.fetch(). let view = repo.view(); assert!(view.heads().contains(&jj_id(&initial_git_commit))); let initial_commit_target = RefTarget::normal(jj_id(&initial_git_commit)); @@ -2350,19 +2361,51 @@ fn test_fetch_initial_commit_head_is_set() { .unwrap(); let mut tx = test_data.repo.start_transaction(&test_data.settings); - let stats = git::fetch( - tx.repo_mut(), - &test_data.git_repo, - "origin", - &[StringPattern::everything()], - git::RemoteCallbacks::default(), - &git_settings, - None, - ) - .unwrap(); + { + let mut git_fetch = GitFetch::new( + tx.repo_mut(), + &test_data.git_repo, + &git_settings, + git::fetch_options(git::RemoteCallbacks::default(), None), + ); - assert_eq!(stats.default_branch, Some("main".to_string())); - assert!(stats.import_stats.abandoned_commits.is_empty()); + let default_branch = git_fetch + .fetch(&[StringPattern::everything()], "origin") + .unwrap(); + let import_stats = git_fetch.import_refs().unwrap(); + + assert_eq!(default_branch, Some("main".to_string())); + assert!(import_stats.abandoned_commits.is_empty()); + } + + let repo = tx.commit("test").unwrap(); + // The new commit visible after git_fetch.fetch(). + let view = repo.view(); + assert!(view.heads().contains(&jj_id(&new_git_commit))); + let commit_target = RefTarget::normal(jj_id(&new_git_commit)); + let commit_remote_ref = RemoteRef { + target: commit_target.clone(), + state: RemoteRefState::Tracking, + }; + assert_eq!( + *view.git_refs(), + btreemap! { + "refs/remotes/origin/main".to_string() => commit_target.clone(), + "refs/tags/v1.0".to_string() => commit_target.clone(), + + } + ); + assert_eq!( + view.bookmarks().collect::>(), + btreemap! { + "main" => BookmarkTarget { + local_target: &commit_target, + remote_refs: vec![ + ("origin", &commit_remote_ref), + ], + }, + } + ); } #[test] @@ -2375,16 +2418,20 @@ fn test_fetch_success() { let initial_git_commit = empty_git_commit(&test_data.origin_repo, "refs/heads/main", &[]); let mut tx = test_data.repo.start_transaction(&test_data.settings); - git::fetch( - tx.repo_mut(), - &test_data.git_repo, - "origin", - &[StringPattern::everything()], - git::RemoteCallbacks::default(), - &git_settings, - None, - ) - .unwrap(); + { + let mut git_fetch = GitFetch::new( + tx.repo_mut(), + &test_data.git_repo, + &git_settings, + git::fetch_options(git::RemoteCallbacks::default(), None), + ); + + git_fetch + .fetch(&[StringPattern::everything()], "origin") + .unwrap(); + git_fetch.import_refs().unwrap(); + } + test_data.repo = tx.commit("test").unwrap(); test_data.origin_repo.set_head("refs/heads/main").unwrap(); @@ -2399,19 +2446,23 @@ fn test_fetch_success() { .unwrap(); let mut tx = test_data.repo.start_transaction(&test_data.settings); - let stats = git::fetch( - tx.repo_mut(), - &test_data.git_repo, - "origin", - &[StringPattern::everything()], - git::RemoteCallbacks::default(), - &git_settings, - None, - ) - .unwrap(); - // The default bookmark is "main" - assert_eq!(stats.default_branch, Some("main".to_string())); - assert!(stats.import_stats.abandoned_commits.is_empty()); + { + let mut git_fetch = GitFetch::new( + tx.repo_mut(), + &test_data.git_repo, + &git_settings, + git::fetch_options(git::RemoteCallbacks::default(), None), + ); + + let default_branch = git_fetch + .fetch(&[StringPattern::everything()], "origin") + .unwrap(); + let import_stats = git_fetch.import_refs().unwrap(); + + assert_eq!(default_branch, Some("main".to_string())); + assert!(import_stats.abandoned_commits.is_empty()); + } + let repo = tx.commit("test").unwrap(); // The new commit is visible after we fetch again let view = repo.view(); @@ -2457,16 +2508,20 @@ fn test_fetch_prune_deleted_ref() { let commit = empty_git_commit(&test_data.origin_repo, "refs/heads/main", &[]); let mut tx = test_data.repo.start_transaction(&test_data.settings); - git::fetch( - tx.repo_mut(), - &test_data.git_repo, - "origin", - &[StringPattern::everything()], - git::RemoteCallbacks::default(), - &git_settings, - None, - ) - .unwrap(); + { + let mut git_fetch = GitFetch::new( + tx.repo_mut(), + &test_data.git_repo, + &git_settings, + git::fetch_options(git::RemoteCallbacks::default(), None), + ); + + git_fetch + .fetch(&[StringPattern::everything()], "origin") + .unwrap(); + git_fetch.import_refs().unwrap(); + } + // Test the setup assert!(tx.repo_mut().get_local_bookmark("main").is_present()); assert!(tx @@ -2481,17 +2536,22 @@ fn test_fetch_prune_deleted_ref() { .delete() .unwrap(); // After re-fetching, the bookmark should be deleted - let stats = git::fetch( - tx.repo_mut(), - &test_data.git_repo, - "origin", - &[StringPattern::everything()], - git::RemoteCallbacks::default(), - &git_settings, - None, - ) - .unwrap(); - assert_eq!(stats.import_stats.abandoned_commits, vec![jj_id(&commit)]); + { + let mut git_fetch = GitFetch::new( + tx.repo_mut(), + &test_data.git_repo, + &git_settings, + git::fetch_options(git::RemoteCallbacks::default(), None), + ); + + git_fetch + .fetch(&[StringPattern::everything()], "origin") + .unwrap(); + let import_stats = git_fetch.import_refs().unwrap(); + + assert_eq!(import_stats.abandoned_commits, vec![jj_id(&commit)]); + } + assert!(tx.repo_mut().get_local_bookmark("main").is_absent()); assert!(tx .repo_mut() @@ -2509,16 +2569,17 @@ fn test_fetch_no_default_branch() { let initial_git_commit = empty_git_commit(&test_data.origin_repo, "refs/heads/main", &[]); let mut tx = test_data.repo.start_transaction(&test_data.settings); - git::fetch( + let mut git_fetch = GitFetch::new( tx.repo_mut(), &test_data.git_repo, - "origin", - &[StringPattern::everything()], - git::RemoteCallbacks::default(), &git_settings, - None, - ) - .unwrap(); + git::fetch_options(git::RemoteCallbacks::default(), None), + ); + + git_fetch + .fetch(&[StringPattern::everything()], "origin") + .unwrap(); + git_fetch.import_refs().unwrap(); empty_git_commit( &test_data.origin_repo, @@ -2533,18 +2594,12 @@ fn test_fetch_no_default_branch() { .set_head_detached(initial_git_commit.id()) .unwrap(); - let stats = git::fetch( - tx.repo_mut(), - &test_data.git_repo, - "origin", - &[StringPattern::everything()], - git::RemoteCallbacks::default(), - &git_settings, - None, - ) - .unwrap(); + let default_branch = git_fetch + .fetch(&[StringPattern::everything()], "origin") + .unwrap(); + git_fetch.import_refs().unwrap(); // There is no default bookmark - assert_eq!(stats.default_branch, None); + assert_eq!(default_branch, None); } #[test] @@ -2555,16 +2610,18 @@ fn test_fetch_empty_refspecs() { // Base refspecs shouldn't be respected let mut tx = test_data.repo.start_transaction(&test_data.settings); - git::fetch( - tx.repo_mut(), - &test_data.git_repo, - "origin", - &[], - git::RemoteCallbacks::default(), - &git_settings, - None, - ) - .unwrap(); + { + let mut git_fetch = GitFetch::new( + tx.repo_mut(), + &test_data.git_repo, + &git_settings, + git::fetch_options(git::RemoteCallbacks::default(), None), + ); + + git_fetch.fetch(&[], "origin").unwrap(); + git_fetch.import_refs().unwrap(); + } + assert!(tx .repo_mut() .get_remote_bookmark("main", "origin") @@ -2582,15 +2639,13 @@ fn test_fetch_no_such_remote() { let test_data = GitRepoData::create(); let git_settings = GitSettings::default(); let mut tx = test_data.repo.start_transaction(&test_data.settings); - let result = git::fetch( + let mut git_fetch = GitFetch::new( tx.repo_mut(), &test_data.git_repo, - "invalid-remote", - &[StringPattern::everything()], - git::RemoteCallbacks::default(), &git_settings, - None, + git::fetch_options(git::RemoteCallbacks::default(), None), ); + let result = git_fetch.fetch(&[StringPattern::everything()], "invalid-remote"); assert!(matches!(result, Err(GitFetchError::NoSuchRemote(_)))); }