Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

index: clean up base data offsets calculation #3086

Merged
merged 2 commits into from
Feb 18, 2024
Merged

index: clean up base data offsets calculation #3086

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0b3d0ab
index: precompute base data offsets
yuja Feb 18, 2024
3f8aa18
index: inline entry size calculation
yuja Feb 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 42 additions & 46 deletions lib/src/default_index/readonly.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,13 +214,16 @@ pub(super) struct ReadonlyIndexSegment {
name: String,
commit_id_length: usize,
change_id_length: usize,
commit_graph_entry_size: usize,
commit_lookup_entry_size: usize,
// Number of commits not counting the parent file
num_local_commits: u32,
num_local_change_ids: u32,
num_parent_overflow_entries: u32,
num_change_overflow_entries: u32,
// Base data offsets in bytes:
commit_lookup_base: usize,
change_id_table_base: usize,
change_pos_table_base: usize,
parent_overflow_base: usize,
change_overflow_base: usize,
data: Vec<u8>,
}

Expand Down Expand Up @@ -318,6 +321,7 @@ impl ReadonlyIndexSegment {
let num_change_overflow_entries = read_u32(file)?;
let mut data = vec![];
file.read_to_end(&mut data).map_err(from_io_err)?;

let commit_graph_entry_size = CommitGraphEntry::size(commit_id_length);
let graph_size = (num_local_commits as usize) * commit_graph_entry_size;
let commit_lookup_entry_size = CommitLookupEntry::size(commit_id_length);
Expand All @@ -326,30 +330,36 @@ impl ReadonlyIndexSegment {
let change_pos_table_size = (num_local_change_ids as usize) * 4;
let parent_overflow_size = (num_parent_overflow_entries as usize) * 4;
let change_overflow_size = (num_change_overflow_entries as usize) * 4;
let expected_size = graph_size
+ commit_lookup_size
+ change_id_table_size
+ change_pos_table_size
+ parent_overflow_size
+ change_overflow_size;

let graph_base = 0;
let commit_lookup_base = graph_base + graph_size;
let change_id_table_base = commit_lookup_base + commit_lookup_size;
let change_pos_table_base = change_id_table_base + change_id_table_size;
let parent_overflow_base = change_pos_table_base + change_pos_table_size;
let change_overflow_base = parent_overflow_base + parent_overflow_size;
let expected_size = change_overflow_base + change_overflow_size;

if data.len() != expected_size {
return Err(ReadonlyIndexLoadError::invalid_data(
name,
"unexpected data length",
));
}

Ok(Arc::new(ReadonlyIndexSegment {
parent_file,
num_parent_commits,
name,
commit_id_length,
change_id_length,
commit_graph_entry_size,
commit_lookup_entry_size,
num_local_commits,
num_local_change_ids,
num_parent_overflow_entries,
num_change_overflow_entries,
commit_lookup_base,
change_id_table_base,
change_pos_table_base,
parent_overflow_base,
change_overflow_base,
data,
}))
}
Expand All @@ -371,19 +381,20 @@ impl ReadonlyIndexSegment {
}

fn graph_entry(&self, local_pos: LocalPosition) -> CommitGraphEntry {
assert!(local_pos.0 < self.num_local_commits);
let offset = (local_pos.0 as usize) * self.commit_graph_entry_size;
let table = &self.data[..self.commit_lookup_base];
let entry_size = CommitGraphEntry::size(self.commit_id_length);
let offset = (local_pos.0 as usize) * entry_size;
CommitGraphEntry {
data: &self.data[offset..][..self.commit_graph_entry_size],
data: &table[offset..][..entry_size],
}
}

fn commit_lookup_entry(&self, lookup_pos: u32) -> CommitLookupEntry {
assert!(lookup_pos < self.num_local_commits);
let offset = (lookup_pos as usize) * self.commit_lookup_entry_size
+ (self.num_local_commits as usize) * self.commit_graph_entry_size;
let table = &self.data[self.commit_lookup_base..self.change_id_table_base];
let entry_size = CommitLookupEntry::size(self.commit_id_length);
let offset = (lookup_pos as usize) * entry_size;
CommitLookupEntry {
data: &self.data[offset..][..self.commit_lookup_entry_size],
data: &table[offset..][..entry_size],
commit_id_length: self.commit_id_length,
}
}
Expand All @@ -394,47 +405,32 @@ impl ReadonlyIndexSegment {

// might be better to add borrowed version of ChangeId
fn change_lookup_id_bytes(&self, lookup_pos: u32) -> &[u8] {
assert!(lookup_pos < self.num_local_change_ids);
let offset = (lookup_pos as usize) * self.change_id_length
+ (self.num_local_commits as usize) * self.commit_graph_entry_size
+ (self.num_local_commits as usize) * self.commit_lookup_entry_size;
&self.data[offset..][..self.change_id_length]
let table = &self.data[self.change_id_table_base..self.change_pos_table_base];
let offset = (lookup_pos as usize) * self.change_id_length;
&table[offset..][..self.change_id_length]
}

fn change_lookup_pos(&self, lookup_pos: u32) -> ChangeLocalPosition {
assert!(lookup_pos < self.num_local_change_ids);
let offset = (lookup_pos as usize) * 4
+ (self.num_local_commits as usize) * self.commit_graph_entry_size
+ (self.num_local_commits as usize) * self.commit_lookup_entry_size
+ (self.num_local_change_ids as usize) * self.change_id_length;
ChangeLocalPosition(u32::from_le_bytes(
self.data[offset..][..4].try_into().unwrap(),
))
let table = &self.data[self.change_pos_table_base..self.parent_overflow_base];
let offset = (lookup_pos as usize) * 4;
ChangeLocalPosition(u32::from_le_bytes(table[offset..][..4].try_into().unwrap()))
}

fn overflow_parents(&self, overflow_pos: u32, num_parents: u32) -> SmallIndexPositionsVec {
assert!(overflow_pos + num_parents <= self.num_parent_overflow_entries);
let offset = (overflow_pos as usize) * 4
+ (self.num_local_commits as usize) * self.commit_graph_entry_size
+ (self.num_local_commits as usize) * self.commit_lookup_entry_size
+ (self.num_local_change_ids as usize) * self.change_id_length
+ (self.num_local_change_ids as usize) * 4;
self.data[offset..][..(num_parents as usize) * 4]
let table = &self.data[self.parent_overflow_base..self.change_overflow_base];
let offset = (overflow_pos as usize) * 4;
let size = (num_parents as usize) * 4;
table[offset..][..size]
.chunks_exact(4)
.map(|chunk| IndexPosition(u32::from_le_bytes(chunk.try_into().unwrap())))
.collect()
}

/// Scans graph entry positions stored in the overflow change ids table.
fn overflow_changes_from(&self, overflow_pos: u32) -> impl Iterator<Item = LocalPosition> + '_ {
let base = (self.num_local_commits as usize) * self.commit_graph_entry_size
+ (self.num_local_commits as usize) * self.commit_lookup_entry_size
+ (self.num_local_change_ids as usize) * self.change_id_length
+ (self.num_local_change_ids as usize) * 4
+ (self.num_parent_overflow_entries as usize) * 4;
let size = (self.num_change_overflow_entries as usize) * 4;
let table = &self.data[self.change_overflow_base..];
let offset = (overflow_pos as usize) * 4;
self.data[base..][..size][offset..]
table[offset..]
.chunks_exact(4)
.map(|chunk| LocalPosition(u32::from_le_bytes(chunk.try_into().unwrap())))
}
Expand Down