From 8e1c7b22e5543dd9cb1ef3b04df9a6032aeced73 Mon Sep 17 00:00:00 2001 From: Oliver Nordbjerg Date: Mon, 4 Sep 2023 11:31:09 +0200 Subject: [PATCH 1/4] feat: add per-table stats --- src/table.rs | 73 +++++++++++++++++++++++++++++++++++++++++ src/tree_store/btree.rs | 9 +++++ 2 files changed, 82 insertions(+) diff --git a/src/table.rs b/src/table.rs index 060f0d1d..506a104f 100644 --- a/src/table.rs +++ b/src/table.rs @@ -10,6 +10,50 @@ use std::borrow::Borrow; use std::ops::RangeBounds; use std::sync::{Arc, Mutex}; +/// Informational storage stats about a table +#[derive(Debug)] +pub struct TableStats { + pub(crate) tree_height: u32, + pub(crate) leaf_pages: u64, + pub(crate) branch_pages: u64, + pub(crate) stored_leaf_bytes: u64, + pub(crate) metadata_bytes: u64, + pub(crate) fragmented_bytes: u64, +} + +impl TableStats { + /// Maximum traversal distance to reach the deepest (key, value) pair, across all tables + pub fn tree_height(&self) -> u32 { + self.tree_height + } + + /// Number of leaf pages that store user data + pub fn leaf_pages(&self) -> u64 { + self.leaf_pages + } + + /// Number of branch pages in btrees that store user data + pub fn branch_pages(&self) -> u64 { + self.branch_pages + } + + /// Number of bytes consumed by keys and values that have been inserted. + /// Does not include indexing overhead + pub fn stored_bytes(&self) -> u64 { + self.stored_leaf_bytes + } + + /// Number of bytes consumed by keys in internal branch pages, plus other metadata + pub fn metadata_bytes(&self) -> u64 { + self.metadata_bytes + } + + /// Number of bytes consumed by fragmentation, both in data pages and internal metadata tables + pub fn fragmented_bytes(&self) -> u64 { + self.fragmented_bytes + } +} + /// A table containing key-value mappings pub struct Table<'db, 'txn, K: RedbKey + 'static, V: RedbValue + 'static> { name: String, @@ -174,6 +218,19 @@ impl<'db, 'txn, K: RedbKey + 'static, V: RedbValue + 'static> ReadableTable Result { + let tree_stats = self.tree.stats()?; + + Ok(TableStats { + tree_height: tree_stats.tree_height, + leaf_pages: tree_stats.leaf_pages, + branch_pages: tree_stats.branch_pages, + stored_leaf_bytes: tree_stats.stored_leaf_bytes, + metadata_bytes: tree_stats.metadata_bytes, + fragmented_bytes: tree_stats.fragmented_bytes, + }) + } + fn len(&self) -> Result { self.tree.len() } @@ -235,6 +292,9 @@ pub trait ReadableTable: Sealed { K: 'a, KR: Borrow> + 'a; + /// Retrieves information about storage usage for the table + fn stats(&self) -> Result; + /// Returns the number of entries in the table fn len(&self) -> Result; @@ -282,6 +342,19 @@ impl<'txn, K: RedbKey + 'static, V: RedbValue + 'static> ReadableTable self.tree.range(&range).map(Range::new) } + fn stats(&self) -> Result { + let tree_stats = self.tree.stats()?; + + Ok(TableStats { + tree_height: tree_stats.tree_height, + leaf_pages: tree_stats.leaf_pages, + branch_pages: tree_stats.branch_pages, + stored_leaf_bytes: tree_stats.stored_leaf_bytes, + metadata_bytes: tree_stats.metadata_bytes, + fragmented_bytes: tree_stats.fragmented_bytes, + }) + } + fn len(&self) -> Result { self.tree.len() } diff --git a/src/tree_store/btree.rs b/src/tree_store/btree.rs index d5aab750..0121e148 100644 --- a/src/tree_store/btree.rs +++ b/src/tree_store/btree.rs @@ -614,6 +614,15 @@ impl<'a, K: RedbKey, V: RedbValue> Btree<'a, K, V> { Ok(count) } + pub(crate) fn stats(&self) -> Result { + btree_stats( + self.root.map(|(p, _)| p), + self.mem, + K::fixed_width(), + V::fixed_width(), + ) + } + #[allow(dead_code)] pub(crate) fn print_debug(&self, include_values: bool) -> Result { if let Some((p, _)) = self.root { From c12663a2bc93b27fc74bcfc0582525a210271a8a Mon Sep 17 00:00:00 2001 From: Oliver Nordbjerg Date: Mon, 4 Sep 2023 11:35:27 +0200 Subject: [PATCH 2/4] feat: add stats for multimap tables --- src/multimap_table.rs | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/multimap_table.rs b/src/multimap_table.rs index 4e94c49b..c75bda67 100644 --- a/src/multimap_table.rs +++ b/src/multimap_table.rs @@ -14,6 +14,7 @@ use std::mem; use std::mem::size_of; use std::ops::{RangeBounds, RangeFull}; use std::sync::{Arc, Mutex}; +use crate::table::TableStats; // Verify all the checksums in the tree, including any Dynamic collection subtrees pub(crate) fn verify_tree_and_subtree_checksums( @@ -922,6 +923,19 @@ impl<'db, 'txn, K: RedbKey + 'static, V: RedbKey + 'static> ReadableMultimapTabl Ok(MultimapRange::new(inner, self.mem)) } + fn stats(&self) -> Result { + let tree_stats = self.tree.stats()?; + + Ok(TableStats { + tree_height: tree_stats.tree_height, + leaf_pages: tree_stats.leaf_pages, + branch_pages: tree_stats.branch_pages, + stored_leaf_bytes: tree_stats.stored_leaf_bytes, + metadata_bytes: tree_stats.metadata_bytes, + fragmented_bytes: tree_stats.fragmented_bytes, + }) + } + /// Returns the number of key-value pairs in the table fn len(&self) -> Result { let mut count = 0; @@ -963,6 +977,9 @@ pub trait ReadableMultimapTable: Sea K: 'a, KR: Borrow> + 'a; + /// Retrieves information about storage usage for the table + fn stats(&self) -> Result; + fn len(&self) -> Result; fn is_empty(&self) -> Result; @@ -1025,6 +1042,19 @@ impl<'txn, K: RedbKey + 'static, V: RedbKey + 'static> ReadableMultimapTable Result { + let tree_stats = self.tree.stats()?; + + Ok(TableStats { + tree_height: tree_stats.tree_height, + leaf_pages: tree_stats.leaf_pages, + branch_pages: tree_stats.branch_pages, + stored_leaf_bytes: tree_stats.stored_leaf_bytes, + metadata_bytes: tree_stats.metadata_bytes, + fragmented_bytes: tree_stats.fragmented_bytes, + }) + } + fn len(&self) -> Result { let mut count = 0; for item in self.iter()? { From d3234b6ee5a477b5fbf3cc78121a19b84f4fde6f Mon Sep 17 00:00:00 2001 From: Oliver Nordbjerg Date: Mon, 4 Sep 2023 11:36:25 +0200 Subject: [PATCH 3/4] docs: adjust docs --- src/table.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/table.rs b/src/table.rs index 506a104f..6420433c 100644 --- a/src/table.rs +++ b/src/table.rs @@ -22,7 +22,7 @@ pub struct TableStats { } impl TableStats { - /// Maximum traversal distance to reach the deepest (key, value) pair, across all tables + /// Maximum traversal distance to reach the deepest (key, value) pair in the table pub fn tree_height(&self) -> u32 { self.tree_height } @@ -32,7 +32,7 @@ impl TableStats { self.leaf_pages } - /// Number of branch pages in btrees that store user data + /// Number of branch pages in the btree that store user data pub fn branch_pages(&self) -> u64 { self.branch_pages } From d5be0c6387172a605dfb03eded28a16b7a59c4d2 Mon Sep 17 00:00:00 2001 From: Oliver Nordbjerg Date: Tue, 5 Sep 2023 13:26:48 +0200 Subject: [PATCH 4/4] chore: fmt --- src/multimap_table.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/multimap_table.rs b/src/multimap_table.rs index c75bda67..e712b4f0 100644 --- a/src/multimap_table.rs +++ b/src/multimap_table.rs @@ -1,5 +1,6 @@ use crate::multimap_table::DynamicCollectionType::{Inline, Subtree}; use crate::sealed::Sealed; +use crate::table::TableStats; use crate::tree_store::{ AllPageNumbersBtreeIter, Btree, BtreeMut, BtreeRangeIter, CachePriority, Checksum, LeafAccessor, LeafMutator, Page, PageHint, PageNumber, RawBtree, RawLeafBuilder, @@ -14,7 +15,6 @@ use std::mem; use std::mem::size_of; use std::ops::{RangeBounds, RangeFull}; use std::sync::{Arc, Mutex}; -use crate::table::TableStats; // Verify all the checksums in the tree, including any Dynamic collection subtrees pub(crate) fn verify_tree_and_subtree_checksums(