From c73f0280c901c321b9b5cdb863ac2ea7d2390e7d Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Tue, 20 Aug 2024 08:32:53 +0100 Subject: [PATCH 1/2] chore: use heed for LMDB in benchmark --- Cargo.toml | 2 +- benches/common.rs | 99 +++++++++++++++---------------- benches/int_benchmark.rs | 10 +++- benches/large_values_benchmark.rs | 10 +++- benches/lmdb_benchmark.rs | 10 +++- 5 files changed, 70 insertions(+), 61 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5ab3f7c5..b37d946f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,7 +40,7 @@ bincode = "1.3.3" # Pinned to compatibility with MSRV ctrlc = "=3.2.3" fastrand = "2.0.0" -lmdb-rkv = "0.14.0" +heed = "0.20" sanakirja = "=1.4.1" sanakirja-core = "=1.4.1" sled = "0.34.7" diff --git a/benches/common.rs b/benches/common.rs index 4d06af1b..00b0f114 100644 --- a/benches/common.rs +++ b/benches/common.rs @@ -4,6 +4,7 @@ use sanakirja::btree::page_unsized; use sanakirja::{Commit, RootDb}; use std::fs; use std::fs::File; +use std::ops::Bound; use std::path::Path; #[allow(dead_code)] @@ -321,123 +322,119 @@ impl<'a> BenchInserter for SledBenchInserter<'a> { } } -pub struct LmdbRkvBenchDatabase<'a> { - env: &'a lmdb::Environment, - db: lmdb::Database, +pub struct HeedBenchDatabase<'a> { + env: &'a heed::Env, + db: heed::Database, } -impl<'a> LmdbRkvBenchDatabase<'a> { - pub fn new(env: &'a lmdb::Environment) -> Self { - let db = env.open_db(None).unwrap(); - LmdbRkvBenchDatabase { env, db } +impl<'a> HeedBenchDatabase<'a> { + pub fn new(env: &'a heed::Env) -> Self { + let mut tx = env.write_txn().unwrap(); + let db = env.create_database(&mut tx, None).unwrap(); + Self { env, db } } } -impl<'a> BenchDatabase for LmdbRkvBenchDatabase<'a> { - type W<'db> = LmdbRkvBenchWriteTransaction<'db> where Self: 'db; - type R<'db> = LmdbRkvBenchReadTransaction<'db> where Self: 'db; +impl<'a> BenchDatabase for HeedBenchDatabase<'a> { + type W<'db> = HeedBenchWriteTransaction<'db> where Self: 'db; + type R<'db> = HeedBenchReadTransaction<'db> where Self: 'db; fn db_type_name() -> &'static str { - "lmdb-rkv" + "heed" } fn write_transaction(&self) -> Self::W<'_> { - let txn = self.env.begin_rw_txn().unwrap(); - LmdbRkvBenchWriteTransaction { db: self.db, txn } + let txn = self.env.write_txn().unwrap(); + Self::W { db: self.db, txn } } fn read_transaction(&self) -> Self::R<'_> { - let txn = self.env.begin_ro_txn().unwrap(); - LmdbRkvBenchReadTransaction { db: self.db, txn } + let txn = self.env.read_txn().unwrap(); + Self::R { db: self.db, txn } } } -pub struct LmdbRkvBenchWriteTransaction<'db> { - db: lmdb::Database, - txn: lmdb::RwTransaction<'db>, +pub struct HeedBenchWriteTransaction<'db> { + db: heed::Database, + txn: heed::RwTxn<'db>, } -impl<'db> BenchWriteTransaction for LmdbRkvBenchWriteTransaction<'db> { - type W<'txn> = LmdbRkvBenchInserter<'txn, 'db> where Self: 'txn; +impl<'db> BenchWriteTransaction for HeedBenchWriteTransaction<'db> { + type W<'txn> = HeedBenchInserter<'txn, 'db> where Self: 'txn; fn get_inserter(&mut self) -> Self::W<'_> { - LmdbRkvBenchInserter { + Self::W { db: self.db, txn: &mut self.txn, } } fn commit(self) -> Result<(), ()> { - use lmdb::Transaction; self.txn.commit().map_err(|_| ()) } } -pub struct LmdbRkvBenchInserter<'txn, 'db> { - db: lmdb::Database, - txn: &'txn mut lmdb::RwTransaction<'db>, +pub struct HeedBenchInserter<'txn, 'db> { + db: heed::Database, + txn: &'txn mut heed::RwTxn<'db>, } -impl BenchInserter for LmdbRkvBenchInserter<'_, '_> { +impl BenchInserter for HeedBenchInserter<'_, '_> { fn insert(&mut self, key: &[u8], value: &[u8]) -> Result<(), ()> { - self.txn - .put(self.db, &key, &value, lmdb::WriteFlags::empty()) - .map_err(|_| ()) + self.db.put(self.txn, key, value).map_err(|_| ()) } fn remove(&mut self, key: &[u8]) -> Result<(), ()> { - self.txn.del(self.db, &key, None).map_err(|_| ()) + self.db.delete(self.txn, key).map(|_| ()).map_err(|_| ()) } } -pub struct LmdbRkvBenchReadTransaction<'db> { - db: lmdb::Database, - txn: lmdb::RoTransaction<'db>, +pub struct HeedBenchReadTransaction<'db> { + db: heed::Database, + txn: heed::RoTxn<'db>, } -impl<'db> BenchReadTransaction for LmdbRkvBenchReadTransaction<'db> { - type T<'txn> = LmdbRkvBenchReader<'txn, 'db> where Self: 'txn; +impl<'db> BenchReadTransaction for HeedBenchReadTransaction<'db> { + type T<'txn> = HeedBenchReader<'txn, 'db> where Self: 'txn; fn get_reader(&self) -> Self::T<'_> { - LmdbRkvBenchReader { + Self::T { db: self.db, txn: &self.txn, } } } -pub struct LmdbRkvBenchReader<'txn, 'db> { - db: lmdb::Database, - txn: &'txn lmdb::RoTransaction<'db>, +pub struct HeedBenchReader<'txn, 'db> { + db: heed::Database, + txn: &'txn heed::RoTxn<'db>, } -impl<'txn, 'db> BenchReader for LmdbRkvBenchReader<'txn, 'db> { +impl<'txn, 'db> BenchReader for HeedBenchReader<'txn, 'db> { type Output<'out> = &'out [u8] where Self: 'out; - type Iterator<'out> = LmdbRkvBenchIterator<'out> where Self: 'out; + type Iterator<'out> = HeedBenchIterator<'out> where Self: 'out; fn get(&self, key: &[u8]) -> Option<&[u8]> { - use lmdb::Transaction; - self.txn.get(self.db, &key).ok() + self.db.get(self.txn, key).unwrap() } fn range_from<'a>(&'a self, key: &'a [u8]) -> Self::Iterator<'a> { - use lmdb::{Cursor, Transaction}; - let iter = self.txn.open_ro_cursor(self.db).unwrap().iter_from(key); + let range = (Bound::Included(key), Bound::Unbounded); + let iter = self.db.range(self.txn, &range).unwrap(); - LmdbRkvBenchIterator { iter } + Self::Iterator { iter } } fn len(&self) -> u64 { - use lmdb::Transaction; - self.txn.stat(self.db).unwrap().entries() as u64 + self.db.stat(self.txn).unwrap().entries as u64 } } -pub struct LmdbRkvBenchIterator<'a> { - iter: lmdb::Iter<'a>, +pub struct HeedBenchIterator<'a> { + iter: heed::RoRange<'a, heed::types::Bytes, heed::types::Bytes>, } -impl BenchIterator for LmdbRkvBenchIterator<'_> { +impl BenchIterator for HeedBenchIterator<'_> { type Output<'out> = &'out [u8] where Self: 'out; fn next(&mut self) -> Option<(Self::Output<'_>, Self::Output<'_>)> { diff --git a/benches/int_benchmark.rs b/benches/int_benchmark.rs index 0f24980f..43c5942e 100644 --- a/benches/int_benchmark.rs +++ b/benches/int_benchmark.rs @@ -65,9 +65,13 @@ fn main() { let lmdb_results = { let tmpfile: TempDir = tempfile::tempdir_in(current_dir().unwrap()).unwrap(); - let env = lmdb::Environment::new().open(tmpfile.path()).unwrap(); - env.set_map_size(10 * 4096 * 1024 * 1024).unwrap(); - let table = LmdbRkvBenchDatabase::new(&env); + let env = unsafe { + heed::EnvOpenOptions::new() + .map_size(10 * 4096 * 1024 * 1024) + .open(tmpfile.path()) + .unwrap() + }; + let table = HeedBenchDatabase::new(&env); benchmark(table) }; diff --git a/benches/large_values_benchmark.rs b/benches/large_values_benchmark.rs index cfd0c62c..06088ba0 100644 --- a/benches/large_values_benchmark.rs +++ b/benches/large_values_benchmark.rs @@ -76,9 +76,13 @@ fn main() { let lmdb_results = { let tmpfile: TempDir = tempfile::tempdir_in(current_dir().unwrap()).unwrap(); - let env = lmdb::Environment::new().open(tmpfile.path()).unwrap(); - env.set_map_size(10 * 4096 * 1024 * 1024).unwrap(); - let table = LmdbRkvBenchDatabase::new(&env); + let env = unsafe { + heed::EnvOpenOptions::new() + .map_size(10 * 4096 * 1024 * 1024) + .open(tmpfile.path()) + .unwrap() + }; + let table = HeedBenchDatabase::new(&env); benchmark(table) }; diff --git a/benches/lmdb_benchmark.rs b/benches/lmdb_benchmark.rs index ed596044..7ed58cc1 100644 --- a/benches/lmdb_benchmark.rs +++ b/benches/lmdb_benchmark.rs @@ -299,9 +299,13 @@ fn main() { let lmdb_results = { let tmpfile: TempDir = tempfile::tempdir_in(&tmpdir).unwrap(); - let env = lmdb::Environment::new().open(tmpfile.path()).unwrap(); - env.set_map_size(4096 * 1024 * 1024).unwrap(); - let table = LmdbRkvBenchDatabase::new(&env); + let env = unsafe { + heed::EnvOpenOptions::new() + .map_size(4096 * 1024 * 1024) + .open(tmpfile.path()) + .unwrap() + }; + let table = HeedBenchDatabase::new(&env); benchmark(table) }; From 0e2b6c67b536533d2f72f861eaed801b1cbc16a1 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Tue, 20 Aug 2024 08:43:30 +0100 Subject: [PATCH 2/2] fix syscall bench --- benches/syscall_benchmark.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/benches/syscall_benchmark.rs b/benches/syscall_benchmark.rs index 4abcde93..18503b30 100644 --- a/benches/syscall_benchmark.rs +++ b/benches/syscall_benchmark.rs @@ -1,6 +1,5 @@ use tempfile::{NamedTempFile, TempDir}; -use lmdb::Transaction; use rand::prelude::SliceRandom; use rand::Rng; use std::env::current_dir; @@ -56,22 +55,28 @@ fn gen_data(count: usize, key_size: usize, value_size: usize) -> Vec<(Vec, V } fn lmdb_bench(path: &Path) { - let env = lmdb::Environment::new().open(path).unwrap(); - env.set_map_size(4096 * 1024 * 1024).unwrap(); + let env = unsafe { + heed::EnvOpenOptions::new() + .map_size(4096 * 1024 * 1024) + .open(path) + .unwrap() + }; let mut pairs = gen_data(1000, KEY_SIZE, VALUE_SIZE); let pairs_len = pairs.len(); - let db = env.open_db(None).unwrap(); + let db: heed::Database = env + .open_database(&env.read_txn().unwrap(), None) + .unwrap() + .unwrap(); { let start = SystemTime::now(); - let mut txn = env.begin_rw_txn().unwrap(); + let mut txn = env.write_txn().unwrap(); { for i in 0..ELEMENTS { let (key, value) = &mut pairs[i % pairs_len]; key[0..8].copy_from_slice(&(i as u64).to_le_bytes()); - txn.put(db, &key, &value, lmdb::WriteFlags::empty()) - .unwrap(); + db.put(&mut txn, key, value).unwrap(); } } txn.commit().unwrap(); @@ -83,7 +88,7 @@ fn lmdb_bench(path: &Path) { let mut key_order: Vec = (0..ELEMENTS).collect(); key_order.shuffle(&mut rand::thread_rng()); - let txn = env.begin_ro_txn().unwrap(); + let txn = env.read_txn().unwrap(); { for _ in 0..ITERATIONS { let start = SystemTime::now(); @@ -92,7 +97,7 @@ fn lmdb_bench(path: &Path) { for &i in &key_order { let (key, value) = &mut pairs[i % pairs_len]; key[0..8].copy_from_slice(&(i as u64).to_le_bytes()); - let result: &[u8] = txn.get(db, &key).unwrap(); + let result: &[u8] = db.get(&txn, key).unwrap().unwrap(); checksum += result[0] as u64; expected_checksum += value[0] as u64; }