Skip to content

Commit

Permalink
fix: zero storage capacity
Browse files Browse the repository at this point in the history
  • Loading branch information
@morph-dev
morph-dev committed May 18, 2024
1 parent 15dd96c commit 4ac42b6
Show file tree
Hide file tree
Showing 2 changed files with 117 additions and 40 deletions.
51 changes: 49 additions & 2 deletions trin-storage/src/versioned/id_indexed_v1/pruning_strategy.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,16 @@ impl PruningStrategy {
return 0;
}

// If storage capacity is 0, prune everything.
if self.config.storage_capacity_bytes == 0 {
debug!(
Db = %self.config.content_type,
"Storage capacity is 0. Pruning everything ({})",
usage_stats.entry_count
);
return usage_stats.entry_count;
}

self.estimate_to_delete_until_target(usage_stats)
.min(self.max_pruning_count)
}
Expand Down Expand Up @@ -192,15 +202,19 @@ mod tests {

use super::*;

const STORAGE_CAPACITY_BYTES: u64 = 1_000_000;
const DEFAULT_STORAGE_CAPACITY_BYTES: u64 = 1_000_000;

fn create_default_pruning_strategy() -> PruningStrategy {
create_pruning_strategy(DEFAULT_STORAGE_CAPACITY_BYTES)
}

fn create_pruning_strategy(storage_capacity_bytes: u64) -> PruningStrategy {
let config = IdIndexedV1StoreConfig {
content_type: ContentType::State,
network: ProtocolId::State,
node_id: NodeId::random(),
node_data_dir: PathBuf::default(),
storage_capacity_bytes: STORAGE_CAPACITY_BYTES,
storage_capacity_bytes,
sql_connection_pool: Pool::new(SqliteConnectionManager::memory()).unwrap(),
distance_fn: DistanceFunction::Xor,
pruning_config: PruningConfig::default(),
Expand Down Expand Up @@ -268,6 +282,39 @@ mod tests {
);
}

#[rstest]
#[case::empty(0, 0, false, false, 0)]
#[case::few_entries(100, 20_000, true, true, 100)]
#[case::many_entries(10_000, 1_000_000, true, true, 10_000)]
fn zero_storage_capacity(
#[case] entry_count: u64,
#[case] total_entry_size_bytes: u64,
#[case] is_usage_above_target_capacity: bool,
#[case] should_prune: bool,
#[case] pruning_count: u64,
) {
let pruning_strategy = create_pruning_strategy(/* storage_capacity_bytes= */ 0);
let usage_stats = UsageStats {
entry_count,
total_entry_size_bytes,
};
assert_eq!(
pruning_strategy.is_usage_above_target_capacity(&usage_stats),
is_usage_above_target_capacity,
"testing is_usage_above_target_capacity"
);
assert_eq!(
pruning_strategy.should_prune(&usage_stats),
should_prune,
"testing should_prune"
);
assert_eq!(
pruning_strategy.get_pruning_count(&usage_stats),
pruning_count,
"testing pruning_count"
);
}

#[test]
fn observe_pruning_duration_optimal() {
let mut pruning_strategy = create_default_pruning_strategy();
Expand Down
106 changes: 68 additions & 38 deletions trin-storage/src/versioned/id_indexed_v1/store.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,20 @@ impl IdIndexedV1Store {
.pruning_strategy
.is_usage_above_target_capacity(&self.usage_stats)
{
debug!(
Db = %self.config.content_type,
"Used capacity ({}) is above target capacity ({}) -> Using distance to farthest for radius",
self.usage_stats.total_entry_size_bytes,
self.pruning_strategy.target_capacity_bytes()
);
self.set_radius_to_farthest()?;
} else if self.config.storage_capacity_bytes == 0 {
debug!(
Db = %self.config.content_type,
"Storage capacity is 0 -> Using ZERO radius",
);
self.radius = Distance::ZERO;
self.metrics.report_radius(self.radius);
} else {
debug!(
Db = %self.config.content_type,
Expand Down Expand Up @@ -410,8 +423,16 @@ impl IdIndexedV1Store {
fn set_radius_to_farthest(&mut self) -> Result<(), ContentStoreError> {
match self.lookup_farthest()? {
None => {
error!(Db = %self.config.content_type, "Farthest not found!");
self.radius = Distance::MAX;
if self.config.storage_capacity_bytes == 0 {
debug!(
Db = %self.config.content_type,
"Farthest not found and storage capacity is 0",
);
self.radius = Distance::ZERO;
} else {
error!(Db = %self.config.content_type, "Farthest not found!");
self.radius = Distance::MAX;
}
}
Some(farthest) => {
self.radius = self.distance_to_content_id(&farthest.content_id);
Expand Down Expand Up @@ -527,15 +548,15 @@ mod tests {
// Storage capacity that stores 10000 items of default size
const STORAGE_CAPACITY_10000_ITEMS: u64 = 10000 * CONTENT_DEFAULT_SIZE_BYTES;

fn create_config(temp_dir: &TempDir) -> IdIndexedV1StoreConfig {
fn create_config(temp_dir: &TempDir, storage_capacity_bytes: u64) -> IdIndexedV1StoreConfig {
IdIndexedV1StoreConfig {
content_type: ContentType::State,
network: ProtocolId::State,
node_id: NodeId::random(),
node_data_dir: temp_dir.path().to_path_buf(),
distance_fn: DistanceFunction::Xor,
sql_connection_pool: setup_sql(temp_dir.path()).unwrap(),
storage_capacity_bytes: STORAGE_CAPACITY_100_ITEMS,
storage_capacity_bytes,
pruning_config: PruningConfig::default(),
}
}
Expand Down Expand Up @@ -588,7 +609,7 @@ mod tests {
#[test]
fn create_empty() -> Result<()> {
let temp_dir = TempDir::new()?;
let config = create_config(&temp_dir);
let config = create_config(&temp_dir, STORAGE_CAPACITY_100_ITEMS);
let store = IdIndexedV1Store::create(ContentType::State, config)?;
assert_eq!(store.usage_stats.entry_count, 0);
assert_eq!(store.usage_stats.total_entry_size_bytes, 0);
Expand All @@ -599,7 +620,7 @@ mod tests {
#[test]
fn create_low_usage() -> Result<()> {
let temp_dir = TempDir::new()?;
let config = create_config(&temp_dir);
let config = create_config(&temp_dir, STORAGE_CAPACITY_100_ITEMS);

let item_count = 20; // 20%
create_and_populate_table(&config, item_count)?;
Expand All @@ -618,7 +639,7 @@ mod tests {
#[test]
fn create_half_full() -> Result<()> {
let temp_dir = TempDir::new()?;
let config = create_config(&temp_dir);
let config = create_config(&temp_dir, STORAGE_CAPACITY_100_ITEMS);

let item_count = 50; // 50%
create_and_populate_table(&config, item_count)?;
Expand All @@ -637,7 +658,7 @@ mod tests {
#[test]
fn create_at_target_capacity() -> Result<()> {
let temp_dir = TempDir::new()?;
let config = create_config(&temp_dir);
let config = create_config(&temp_dir, STORAGE_CAPACITY_100_ITEMS);

let target_capacity_bytes = PruningStrategy::new(config.clone()).target_capacity_bytes();
let target_capacity_count = target_capacity_bytes / CONTENT_DEFAULT_SIZE_BYTES;
Expand All @@ -656,7 +677,7 @@ mod tests {
#[test]
fn create_above_target_capacity() -> Result<()> {
let temp_dir = TempDir::new()?;
let config = create_config(&temp_dir);
let config = create_config(&temp_dir, STORAGE_CAPACITY_100_ITEMS);

let target_capacity_bytes = PruningStrategy::new(config.clone()).target_capacity_bytes();
let above_target_capacity_count = 1 + target_capacity_bytes / CONTENT_DEFAULT_SIZE_BYTES;
Expand All @@ -680,7 +701,7 @@ mod tests {
#[test]
fn create_at_full_capacity() -> Result<()> {
let temp_dir = TempDir::new()?;
let config = create_config(&temp_dir);
let config = create_config(&temp_dir, STORAGE_CAPACITY_100_ITEMS);

let full_capacity_count = config.storage_capacity_bytes / CONTENT_DEFAULT_SIZE_BYTES;

Expand All @@ -703,7 +724,7 @@ mod tests {
#[test]
fn create_above_full_capacity() -> Result<()> {
let temp_dir = TempDir::new()?;
let config = create_config(&temp_dir);
let config = create_config(&temp_dir, STORAGE_CAPACITY_100_ITEMS);

let above_full_capacity_count =
10 + config.storage_capacity_bytes / CONTENT_DEFAULT_SIZE_BYTES;
Expand All @@ -725,10 +746,37 @@ mod tests {
Ok(())
}

#[test]
fn create_zero_storage_empty() -> Result<()> {
let temp_dir = TempDir::new()?;
let config = create_config(&temp_dir, /* storage_capacity_bytes= */ 0);

let store = IdIndexedV1Store::create(ContentType::State, config)?;

assert_eq!(store.usage_stats, UsageStats::default());
assert_eq!(store.radius(), Distance::ZERO);
Ok(())
}

#[test]
fn create_zero_storage_non_empty() -> Result<()> {
let temp_dir = TempDir::new()?;
let config = create_config(&temp_dir, /* storage_capacity_bytes= */ 0);

// Add 1K entries, more than we would normally prune.
create_and_populate_table(&config, 1_000)?;
let store = IdIndexedV1Store::create(ContentType::State, config)?;

// Check that db is empty and distance is ZERO.
assert_eq!(store.usage_stats, UsageStats::default());
assert_eq!(store.radius(), Distance::ZERO);
Ok(())
}

#[test]
fn simple_insert_and_lookup() -> Result<()> {
let temp_dir = TempDir::new()?;
let config = create_config(&temp_dir);
let config = create_config(&temp_dir, STORAGE_CAPACITY_100_ITEMS);

// fill 50% of storage with 50 items, 1% each
create_and_populate_table(&config, 50)?;
Expand Down Expand Up @@ -758,7 +806,7 @@ mod tests {
#[test]
fn simple_insert_and_delete() -> Result<()> {
let temp_dir = TempDir::new()?;
let config = create_config(&temp_dir);
let config = create_config(&temp_dir, STORAGE_CAPACITY_100_ITEMS);

// fill 50% of storage with 50 items, 1% each
create_and_populate_table(&config, 50)?;
Expand Down Expand Up @@ -788,7 +836,7 @@ mod tests {
#[test]
fn prune_simple() -> Result<()> {
let temp_dir = TempDir::new()?;
let config = create_config(&temp_dir);
let config = create_config(&temp_dir, STORAGE_CAPACITY_100_ITEMS);
let mut store = IdIndexedV1Store::create(ContentType::State, config.clone())?;

assert_eq!(store.radius(), Distance::MAX);
Expand Down Expand Up @@ -850,7 +898,7 @@ mod tests {
#[test]
fn prune_different_sizes_elements() -> Result<()> {
let temp_dir = TempDir::new()?;
let config = create_config(&temp_dir);
let config = create_config(&temp_dir, STORAGE_CAPACITY_100_ITEMS);

// fill 50% of storage with 50 items, 1% each
create_and_populate_table(&config, 50)?;
Expand Down Expand Up @@ -902,7 +950,7 @@ mod tests {
#[test]
fn prune_with_one_large_item() -> Result<()> {
let temp_dir = TempDir::new()?;
let config = create_config(&temp_dir);
let config = create_config(&temp_dir, STORAGE_CAPACITY_100_ITEMS);

// fill 50% of storage with 50 items, 1% each
create_and_populate_table(&config, 50)?;
Expand Down Expand Up @@ -948,16 +996,7 @@ mod tests {
#[test]
fn prune_big_db() -> Result<()> {
let temp_dir = TempDir::new()?;
let config = IdIndexedV1StoreConfig {
content_type: ContentType::State,
network: ProtocolId::State,
node_id: NodeId::random(),
node_data_dir: temp_dir.path().to_path_buf(),
distance_fn: DistanceFunction::Xor,
sql_connection_pool: setup_sql(temp_dir.path()).unwrap(),
storage_capacity_bytes: STORAGE_CAPACITY_10000_ITEMS,
pruning_config: PruningConfig::default(),
};
let config = create_config(&temp_dir, STORAGE_CAPACITY_10000_ITEMS);

let mut store = IdIndexedV1Store::create(ContentType::State, config.clone())?;

Expand Down Expand Up @@ -996,16 +1035,7 @@ mod tests {
#[test]
fn prune_big_db_with_big_entry() -> Result<()> {
let temp_dir = TempDir::new()?;
let config = IdIndexedV1StoreConfig {
content_type: ContentType::State,
network: ProtocolId::State,
node_id: NodeId::random(),
node_data_dir: temp_dir.path().to_path_buf(),
distance_fn: DistanceFunction::Xor,
sql_connection_pool: setup_sql(temp_dir.path()).unwrap(),
storage_capacity_bytes: STORAGE_CAPACITY_10000_ITEMS,
pruning_config: PruningConfig::default(),
};
let config = create_config(&temp_dir, STORAGE_CAPACITY_10000_ITEMS);

let mut store = IdIndexedV1Store::create(ContentType::State, config.clone())?;

Expand Down Expand Up @@ -1052,7 +1082,7 @@ mod tests {
#[test]
fn pagination_empty() -> Result<()> {
let temp_dir = TempDir::new()?;
let config = create_config(&temp_dir);
let config = create_config(&temp_dir, STORAGE_CAPACITY_100_ITEMS);
let store = IdIndexedV1Store::create(ContentType::State, config)?;

assert_eq!(
Expand All @@ -1068,7 +1098,7 @@ mod tests {
#[test]
fn pagination() -> Result<()> {
let temp_dir = TempDir::new()?;
let config = create_config(&temp_dir);
let config = create_config(&temp_dir, STORAGE_CAPACITY_100_ITEMS);
let mut store = IdIndexedV1Store::create(ContentType::State, config.clone())?;

let entry_count = 12;
Expand Down

0 comments on commit 4ac42b6

Please sign in to comment.