git: refactor jj git clone|fetch to use new GitFetch api directly.

* Make the new `GitFetch` api public.
* Move `git::fetch` to `lib/tests/test_git.rs` as `git_fetch`, to minimize
  churn in the tests. Update test call sites to use `git_fetch`
* Delete the `git::fetch` from `lib/src/git.rs`.
* Update `jj git clone` and `git_fetch` in `cli/src/git_utils.rs` to use
  the new api directly. Removing one redundant layer of indirection.
* This fixes #4920 as it first fetches from all remotes before `import_refs()`
  is called, so there is no race condition if the same commit is treated
  differently in different remotes specified in the same command.
* Add test for the race condition.

Fixes: #4920

CHANGELOG.md

@@ -80,6 +80,11 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 * `jj config unset <TABLE-NAME>` no longer removes a table (such as `[ui]`.)
 
+* `jj git fetch` with multiple remotes will now fetch from all remotes before
+  importing refs into the jj repo. This fixes a race condition where the treatment
+  of a commit that is found in multiple fetch remotes depended on the order the
+  remotes where specified.
+
 ## [0.23.0] - 2024-11-06
 
 ### Security fixes

cli/src/commands/git/clone.rs

@@ -21,6 +21,7 @@ use std::path::PathBuf;
 
 use jj_lib::config::ConfigNamePathBuf;
 use jj_lib::git;
+use jj_lib::git::GitFetch;
 use jj_lib::git::GitFetchError;
 use jj_lib::git::GitFetchStats;
 use jj_lib::repo::Repo;
@@ -221,27 +222,34 @@ fn do_git_clone(
     git_repo.remote(remote_name, source).unwrap();
     let mut fetch_tx = workspace_command.start_transaction();
 
-    let stats = with_remote_git_callbacks(ui, None, |cb| {
-        git::fetch(
+    let stats = with_remote_git_callbacks(ui, None, |cb| -> Result<GitFetchStats, CommandError> {
+        let git_settings = command.settings().git_settings();
+        let mut git_fetch = GitFetch::new(
             fetch_tx.repo_mut(),
             &git_repo,
-            remote_name,
-            &[StringPattern::everything()],
-            cb,
-            &command.settings().git_settings(),
-            depth,
-        )
-    })
-    .map_err(|err| match err {
-        GitFetchError::NoSuchRemote(_) => {
-            panic!("shouldn't happen as we just created the git remote")
-        }
-        GitFetchError::GitImportError(err) => CommandError::from(err),
-        GitFetchError::InternalGitError(err) => map_git_error(err),
-        GitFetchError::InvalidBranchPattern => {
-            unreachable!("we didn't provide any globs")
-        }
+            &git_settings,
+            git::fetch_options(cb, depth),
+        );
+
+        let default_branch = git_fetch
+            .fetch(remote_name, &[StringPattern::everything()])
+            .map_err(|err| match err {
+                GitFetchError::NoSuchRemote(_) => {
+                    panic!("shouldn't happen as we just created the git remote")
+                }
+                GitFetchError::GitImportError(err) => CommandError::from(err),
+                GitFetchError::InternalGitError(err) => map_git_error(err),
+                GitFetchError::InvalidBranchPattern => {
+                    unreachable!("we didn't provide any globs")
+                }
+            })?;
+        let import_stats = git_fetch.import_refs()?;
+        Ok(GitFetchStats {
+            default_branch,
+            import_stats,
+        })
     })?;
+
     print_git_import_stats(ui, fetch_tx.repo(), &stats.import_stats, true)?;
     fetch_tx.finish(ui, "fetch from git remote into empty repo")?;
     Ok((workspace_command, stats))

cli/src/git_util.rs

@@ -27,6 +27,7 @@ use itertools::Itertools;
 use jj_lib::git;
 use jj_lib::git::FailedRefExport;
 use jj_lib::git::FailedRefExportReason;
+use jj_lib::git::GitFetch;
 use jj_lib::git::GitFetchError;
 use jj_lib::git::GitImportStats;
 use jj_lib::git::RefName;
@@ -456,47 +457,49 @@ export or their "parent" bookmarks."#,
     Ok(())
 }
 
+// TODO: Move this back to cli/src/commands/git/fetch.rs
+// With the new `GitFetch` api, this function is too specialized
+// to the `jj git fetch` command and should not be reused.
 pub fn git_fetch(
     ui: &mut Ui,
     tx: &mut WorkspaceCommandTransaction,
     git_repo: &git2::Repository,
     remotes: &[String],
     branch: &[StringPattern],
 ) -> Result<(), CommandError> {
-    let git_settings = tx.settings().git_settings();
-
-    for remote in remotes {
-        let stats = with_remote_git_callbacks(ui, None, |cb| {
-            git::fetch(
+    let import_stats =
+        with_remote_git_callbacks(ui, None, |cb| -> Result<GitImportStats, CommandError> {
+            let git_settings = tx.settings().git_settings();
+            let mut git_fetch = GitFetch::new(
                 tx.repo_mut(),
                 git_repo,
-                remote,
-                branch,
-                cb,
                 &git_settings,
-                None,
-            )
-        })
-        .map_err(|err| match err {
-            GitFetchError::InvalidBranchPattern => {
-                if branch
-                    .iter()
-                    .any(|pattern| pattern.as_exact().is_some_and(|s| s.contains('*')))
-                {
-                    user_error_with_hint(
-                        "Branch names may not include `*`.",
-                        "Prefix the pattern with `glob:` to expand `*` as a glob",
-                    )
-                } else {
-                    user_error(err)
-                }
+                git::fetch_options(cb, None),
+            );
+
+            for remote in remotes {
+                git_fetch.fetch(remote, branch).map_err(|err| match err {
+                    GitFetchError::InvalidBranchPattern => {
+                        if branch
+                            .iter()
+                            .any(|pattern| pattern.as_exact().is_some_and(|s| s.contains('*')))
+                        {
+                            user_error_with_hint(
+                                "Branch names may not include `*`.",
+                                "Prefix the pattern with `glob:` to expand `*` as a glob",
+                            )
+                        } else {
+                            user_error(err)
+                        }
+                    }
+                    GitFetchError::GitImportError(err) => err.into(),
+                    GitFetchError::InternalGitError(err) => map_git_error(err),
+                    _ => user_error(err),
+                })?;
             }
-            GitFetchError::GitImportError(err) => err.into(),
-            GitFetchError::InternalGitError(err) => map_git_error(err),
-            _ => user_error(err),
+            Ok(git_fetch.import_refs()?)
         })?;
-        print_git_import_stats(ui, tx.repo(), &stats.import_stats, true)?;
-    }
+    print_git_import_stats(ui, tx.repo(), &import_stats, true)?;
     warn_if_branches_not_found(
         ui,
         tx,

cli/tests/test_git_fetch.rs

@@ -226,6 +226,29 @@ fn test_git_fetch_multiple_remotes_from_config() {
     "###);
 }
 
+#[test]
+fn test_git_fetch_multiple_remotes_with_same_commit_no_race_condition() {
+    let test_env = TestEnvironment::default();
+    test_env.add_config("git.auto-local-bookmark = true");
+
+    // Create first remote.
+    test_env.jj_cmd_ok(test_env.env_root(), &["git", "init", "repo"]);
+    let repo_path = test_env.env_root().join("repo");
+    add_git_remote(&test_env, &repo_path, "rem1");
+    add_git_remote(&test_env, &repo_path, "rem2");
+
+    test_env.jj_cmd_ok(
+        &repo_path,
+        &["git", "fetch", "--remote", "rem1", "--remote", "rem2"],
+    );
+    insta::assert_snapshot!(get_bookmark_output(&test_env, &repo_path), @r###"
+    rem1: qxosxrvv 6a211027 message
+      @rem1: qxosxrvv 6a211027 message
+    rem2: yszkquru 2497a8a0 message
+      @rem2: yszkquru 2497a8a0 message
+    "###);
+}
+
 #[test]
 fn test_git_fetch_nonexistent_remote() {
     let test_env = TestEnvironment::default();
@@ -237,10 +260,7 @@ fn test_git_fetch_nonexistent_remote() {
         &repo_path,
         &["git", "fetch", "--remote", "rem1", "--remote", "rem2"],
     );
-    insta::assert_snapshot!(stderr, @r###"
-    bookmark: rem1@rem1 [new] untracked
-    Error: No git remote named 'rem2'
-    "###);
+    insta::assert_snapshot!(stderr, @"Error: No git remote named 'rem2'");
     // No remote should have been fetched as part of the failing transaction
     insta::assert_snapshot!(get_bookmark_output(&test_env, &repo_path), @"");
 }
@@ -254,10 +274,7 @@ fn test_git_fetch_nonexistent_remote_from_config() {
     test_env.add_config(r#"git.fetch = ["rem1", "rem2"]"#);
 
     let stderr = &test_env.jj_cmd_failure(&repo_path, &["git", "fetch"]);
-    insta::assert_snapshot!(stderr, @r###"
-    bookmark: rem1@rem1 [new] untracked
-    Error: No git remote named 'rem2'
-    "###);
+    insta::assert_snapshot!(stderr, @"Error: No git remote named 'rem2'");
     // No remote should have been fetched as part of the failing transaction
     insta::assert_snapshot!(get_bookmark_output(&test_env, &repo_path), @"");
 }

lib/src/git.rs

@@ -1229,7 +1229,7 @@ pub enum GitFetchError {
     InternalGitError(#[from] git2::Error),
 }
 
-fn fetch_options(
+pub fn fetch_options(
     callbacks: RemoteCallbacks<'_>,
     depth: Option<NonZeroU32>,
 ) -> git2::FetchOptions<'_> {
@@ -1247,11 +1247,11 @@ fn fetch_options(
 }
 
 struct FetchedBranches {
-    branches: Vec<StringPattern>,
     remote: String,
+    branches: Vec<StringPattern>,
 }
 
-struct GitFetch<'a> {
+pub struct GitFetch<'a> {
     mut_repo: &'a mut MutableRepo,
     git_repo: &'a git2::Repository,
     git_settings: &'a GitSettings,
@@ -1260,7 +1260,7 @@ struct GitFetch<'a> {
 }
 
 impl<'a> GitFetch<'a> {
-    fn new(
+    pub fn new(
         mut_repo: &'a mut MutableRepo,
         git_repo: &'a git2::Repository,
         git_settings: &'a GitSettings,
@@ -1280,10 +1280,10 @@ impl<'a> GitFetch<'a> {
     ///
     /// Keeps track of the {branch_names, remote_name} pair the refs can be
     /// subsequently imported into the `jj` repo by calling `import_refs()`.
-    fn fetch(
+    pub fn fetch(
         &mut self,
-        branch_names: &[StringPattern],
         remote_name: &str,
+        branch_names: &[StringPattern],
     ) -> Result<Option<String>, GitFetchError> {
         let mut remote = self.git_repo.find_remote(remote_name).map_err(|err| {
             if is_remote_not_found_err(&err) {
@@ -1326,8 +1326,8 @@ impl<'a> GitFetch<'a> {
         )?;
 
         self.fetched.push(FetchedBranches {
-            branches: branch_names.to_vec(),
             remote: remote_name.to_string(),
+            branches: branch_names.to_vec(),
         });
 
         // TODO: We could make it optional to get the default branch since we only care
@@ -1394,31 +1394,6 @@ pub struct GitFetchStats {
     pub import_stats: GitImportStats,
 }
 
-#[tracing::instrument(skip(mut_repo, git_repo, callbacks))]
-pub fn fetch(
-    mut_repo: &mut MutableRepo,
-    git_repo: &git2::Repository,
-    remote_name: &str,
-    branch_names: &[StringPattern],
-    callbacks: RemoteCallbacks<'_>,
-    git_settings: &GitSettings,
-    depth: Option<NonZeroU32>,
-) -> Result<GitFetchStats, GitFetchError> {
-    let mut git_fetch = GitFetch::new(
-        mut_repo,
-        git_repo,
-        git_settings,
-        fetch_options(callbacks, depth),
-    );
-    let default_branch = git_fetch.fetch(branch_names, remote_name)?;
-    let import_stats = git_fetch.import_refs()?;
-    let stats = GitFetchStats {
-        default_branch,
-        import_stats,
-    };
-    Ok(stats)
-}
-
 #[derive(Error, Debug, PartialEq)]
 pub enum GitPushError {
     #[error("No git remote named '{0}'")]