index: build and use persistent change id lookup table (with non-lazy reachability test)

cli/tests/test_log_command.rs

@@ -606,10 +606,10 @@ fn test_log_prefix_highlight_counts_hidden_commits() {
     insta::assert_snapshot!(
         test_env.jj_cmd_success(&repo_path, &["log", "-T", prefix_format]),
         @r###"
-    @  Change w[qnwkozpkust] 44[4c3c5066d3]
-    │ ◉  Change q[pvuntsmwlqt] initial ba[1a30916d29] original
+    @  Change wq[nwkozpkust] 44[4c3c5066d3]
+    │ ◉  Change qpv[untsmwlqt] initial ba[1a30916d29] original
     ├─╯
-    ◉  Change z[zzzzzzzzzzz] 00[0000000000]
+    ◉  Change zzz[zzzzzzzzz] 00[0000000000]
     "###
     );
     insta::assert_snapshot!(
@@ -621,7 +621,7 @@ fn test_log_prefix_highlight_counts_hidden_commits() {
     insta::assert_snapshot!(
         test_env.jj_cmd_success(&repo_path, &["log", "-r", "44", "-T", prefix_format]),
         @r###"
-    @  Change w[qnwkozpkust] 44[4c3c5066d3]
+    @  Change wq[nwkozpkust] 44[4c3c5066d3]
     │
     ~
     "###

lib/src/default_index/composite.rs

@@ -23,13 +23,13 @@ use itertools::Itertools;
 
 use super::entry::{
     IndexEntry, IndexPosition, IndexPositionByGeneration, LocalPosition, SmallIndexPositionsVec,
+    SmallLocalPositionsVec,
 };
 use super::readonly::ReadonlyIndexSegment;
 use super::rev_walk::RevWalk;
 use super::revset_engine;
 use crate::backend::{ChangeId, CommitId};
 use crate::hex_util;
-use crate::id_prefix::{IdIndex, IdIndexSource, IdIndexSourceEntry};
 use crate::index::{AllHeadsForGcUnsupported, ChangeIdIndex, Index};
 use crate::object_id::{HexPrefix, ObjectId, PrefixResolution};
 use crate::revset::{ResolvedExpression, Revset, RevsetEvaluationError};
@@ -55,6 +55,16 @@ pub(super) trait IndexSegment: Send + Sync {
 
     fn resolve_commit_id_prefix(&self, prefix: &HexPrefix) -> PrefixResolution<CommitId>;
 
+    fn resolve_neighbor_change_ids(
+        &self,
+        change_id: &ChangeId,
+    ) -> (Option<ChangeId>, Option<ChangeId>);
+
+    fn resolve_change_id_prefix(
+        &self,
+        prefix: &HexPrefix,
+    ) -> PrefixResolution<(ChangeId, SmallLocalPositionsVec)>;
+
     fn generation_number(&self, local_pos: LocalPosition) -> u32;
 
     fn commit_id(&self, local_pos: LocalPosition) -> CommitId;
@@ -189,6 +199,70 @@ impl<'a> CompositeIndex<'a> {
             .unwrap()
     }
 
+    /// Suppose the given `change_id` exists, returns the minimum prefix length
+    /// to disambiguate it within all the indexed ids including hidden ones.
+    pub(super) fn shortest_unique_change_id_prefix_len(&self, change_id: &ChangeId) -> usize {
+        let (prev_id, next_id) = self.resolve_neighbor_change_ids(change_id);
+        itertools::chain(prev_id, next_id)
+            .map(|id| hex_util::common_hex_len(change_id.as_bytes(), id.as_bytes()) + 1)
+            .max()
+            .unwrap_or(0)
+    }
+
+    /// Suppose the given `change_id` exists, returns the previous and next
+    /// change ids in lexicographical order. The returned change ids may be
+    /// hidden.
+    pub(super) fn resolve_neighbor_change_ids(
+        &self,
+        change_id: &ChangeId,
+    ) -> (Option<ChangeId>, Option<ChangeId>) {
+        self.ancestor_index_segments()
+            .map(|segment| segment.resolve_neighbor_change_ids(change_id))
+            .reduce(|(acc_prev_id, acc_next_id), (prev_id, next_id)| {
+                (
+                    acc_prev_id.into_iter().chain(prev_id).max(),
+                    acc_next_id.into_iter().chain(next_id).min(),
+                )
+            })
+            .unwrap()
+    }
+
+    /// Resolves the given change id `prefix` to the associated entries. The
+    /// returned entries may be hidden.
+    ///
+    /// The returned index positions are sorted in ascending order.
+    pub(super) fn resolve_change_id_prefix(
+        &self,
+        prefix: &HexPrefix,
+    ) -> PrefixResolution<(ChangeId, SmallIndexPositionsVec)> {
+        use PrefixResolution::*;
+        self.ancestor_index_segments()
+            .fold(NoMatch, |acc_match, segment| {
+                if acc_match == AmbiguousMatch {
+                    return acc_match; // avoid checking the parent file(s)
+                }
+                let to_global_pos = {
+                    let num_parent_commits = segment.num_parent_commits();
+                    move |LocalPosition(pos)| IndexPosition(pos + num_parent_commits)
+                };
+                // Similar to PrefixResolution::plus(), but merges matches of the same id.
+                match (acc_match, segment.resolve_change_id_prefix(prefix)) {
+                    (NoMatch, local_match) => local_match.map(|(id, positions)| {
+                        (id, positions.into_iter().map(to_global_pos).collect())
+                    }),
+                    (acc_match, NoMatch) => acc_match,
+                    (AmbiguousMatch, _) => AmbiguousMatch,
+                    (_, AmbiguousMatch) => AmbiguousMatch,
+                    (SingleMatch((id1, _)), SingleMatch((id2, _))) if id1 != id2 => AmbiguousMatch,
+                    (SingleMatch((id, mut acc_positions)), SingleMatch((_, local_positions))) => {
+                        acc_positions
+                            .insert_many(0, local_positions.into_iter().map(to_global_pos));
+                        SingleMatch((id, acc_positions))
+                    }
+                }
+            })
+    }
+
     pub(super) fn is_ancestor_pos(
         &self,
         ancestor_pos: IndexPosition,
@@ -422,50 +496,73 @@ impl Index for CompositeIndex<'_> {
 
 pub(super) struct ChangeIdIndexImpl<I> {
     index: I,
-    pos_by_change: IdIndex<ChangeId, IndexPosition, 4>,
+    reachable_bitset: Vec<u64>,
 }
 
 impl<I: AsCompositeIndex> ChangeIdIndexImpl<I> {
     pub fn new(index: I, heads: &mut dyn Iterator<Item = &CommitId>) -> ChangeIdIndexImpl<I> {
-        let mut pos_by_change = IdIndex::builder();
+        // TODO: Calculate reachable bitset lazily.
         let composite = index.as_composite();
+        let bitset_len =
+            usize::try_from(u32::div_ceil(composite.num_commits(), u64::BITS)).unwrap();
+        let mut reachable_bitset = vec![0; bitset_len]; // request zeroed page
         let head_positions = heads
             .map(|id| composite.commit_id_to_pos(id).unwrap())
             .collect_vec();
         for entry in composite.walk_revs(&head_positions, &[]) {
-            pos_by_change.insert(&entry.change_id(), entry.position());
+            let IndexPosition(pos) = entry.position();
+            let bitset_pos = pos / u64::BITS;
+            let bit = 1_u64 << (pos % u64::BITS);
+            reachable_bitset[usize::try_from(bitset_pos).unwrap()] |= bit;
         }
-        Self {
+        ChangeIdIndexImpl {
             index,
-            pos_by_change: pos_by_change.build(),
+            reachable_bitset,
         }
     }
 }
 
 impl<I: AsCompositeIndex + Send + Sync> ChangeIdIndex for ChangeIdIndexImpl<I> {
+    /// Resolves change id prefix among all ids, then filters out hidden
+    /// entries.
+    ///
+    /// If `SingleMatch` is returned, the commits including in the set are all
+    /// visible. `AmbiguousMatch` may be returned even if the prefix is unique
+    /// within the visible entries.
     fn resolve_prefix(&self, prefix: &HexPrefix) -> PrefixResolution<Vec<CommitId>> {
-        self.pos_by_change
-            .resolve_prefix_with(self.index.as_composite(), prefix, |entry| entry.commit_id())
-            .map(|(_, commit_ids)| commit_ids)
+        let index = self.index.as_composite();
+        match index.resolve_change_id_prefix(prefix) {
+            PrefixResolution::NoMatch => PrefixResolution::NoMatch,
+            PrefixResolution::SingleMatch((_change_id, positions)) => {
+                let reachable_commit_ids = positions
+                    .iter()
+                    .filter(|IndexPosition(pos)| {
+                        let bitset_pos = pos / u64::BITS;
+                        let bit = 1_u64 << (pos % u64::BITS);
+                        let bits = self.reachable_bitset[usize::try_from(bitset_pos).unwrap()];
+                        bits & bit != 0
+                    })
+                    .map(|&pos| index.entry_by_pos(pos).commit_id())
+                    .collect_vec();
+                if reachable_commit_ids.is_empty() {
+                    PrefixResolution::NoMatch
+                } else {
+                    PrefixResolution::SingleMatch(reachable_commit_ids)
+                }
+            }
+            PrefixResolution::AmbiguousMatch => PrefixResolution::AmbiguousMatch,
+        }
     }
 
+    /// Calculates the shortest prefix length of the given `change_id` among all
+    /// ids including hidden entries.
+    ///
+    /// The returned length is usually a few digits longer than the minimum
+    /// length to disambiguate within the visible entries.
     fn shortest_unique_prefix_len(&self, change_id: &ChangeId) -> usize {
-        self.pos_by_change
-            .shortest_unique_prefix_len(self.index.as_composite(), change_id)
-    }
-}
-
-impl<'index> IdIndexSource<IndexPosition> for CompositeIndex<'index> {
-    type Entry = IndexEntry<'index>;
-
-    fn entry_at(&self, pointer: &IndexPosition) -> Self::Entry {
-        self.entry_by_pos(*pointer)
-    }
-}
-
-impl IdIndexSourceEntry<ChangeId> for IndexEntry<'_> {
-    fn to_key(&self) -> ChangeId {
-        self.change_id()
+        self.index
+            .as_composite()
+            .shortest_unique_change_id_prefix_len(change_id)
     }
 }
 

lib/src/default_index/entry.rs

@@ -38,6 +38,7 @@ pub(super) struct LocalPosition(pub(super) u32);
 // SmallVec reuses two pointer-size fields as inline area, which meas we can
 // inline up to 16 bytes (on 64-bit platform) for free.
 pub(super) type SmallIndexPositionsVec = SmallVec<[IndexPosition; 4]>;
+pub(super) type SmallLocalPositionsVec = SmallVec<[LocalPosition; 4]>;
 
 #[derive(Clone)]
 pub struct IndexEntry<'a> {