Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove mmap backend #536

Merged
merged 4 commits into from
Mar 24, 2023
Merged

Remove mmap backend #536

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
25903c5
Remove mmap backend
cberner Mar 24, 2023
089164d
Remove PhysicalStorage trait
cberner Mar 24, 2023
afa7a50
Remove nearly all unsafe code
cberner Mar 24, 2023
b15f95c
Remove PageHack and PageHackMut
cberner Mar 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 1 addition & 21 deletions benches/int_benchmark.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,17 +63,6 @@ fn main() {
benchmark(table)
};

let redb_mmap_results = {
let tmpfile: NamedTempFile = NamedTempFile::new_in(current_dir().unwrap()).unwrap();
let db = unsafe {
redb::Database::builder()
.create_mmapped(tmpfile.path())
.unwrap()
};
let table = RedbBenchDatabase::new(&db);
benchmark(table)
};

let lmdb_results = {
let tmpfile: TempDir = tempfile::tempdir_in(current_dir().unwrap()).unwrap();
let env = lmdb::Environment::new().open(tmpfile.path()).unwrap();
Expand Down Expand Up @@ -112,7 +101,6 @@ fn main() {

for results in [
redb_results,
redb_mmap_results,
lmdb_results,
rocksdb_results,
sled_results,
Expand All @@ -125,15 +113,7 @@ fn main() {

let mut table = comfy_table::Table::new();
table.set_width(100);
table.set_header([
"",
"redb",
"redb-mmap",
"lmdb",
"rocksdb",
"sled",
"sanakirja",
]);
table.set_header(["", "redb", "lmdb", "rocksdb", "sled", "sanakirja"]);
for row in rows {
table.add_row(row);
}
Expand Down
37 changes: 1 addition & 36 deletions benches/large_values_benchmark.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,30 +88,6 @@ fn main() {
benchmark(table)
};

let redb_mmap_latency_results = {
let tmpfile: NamedTempFile = NamedTempFile::new_in(current_dir().unwrap()).unwrap();
let db = unsafe {
redb::Database::builder()
.set_write_strategy(WriteStrategy::Checksum)
.create_mmapped(tmpfile.path())
.unwrap()
};
let table = RedbBenchDatabase::new(&db);
benchmark(table)
};

let redb_mmap_throughput_results = {
let tmpfile: NamedTempFile = NamedTempFile::new_in(current_dir().unwrap()).unwrap();
let db = unsafe {
redb::Database::builder()
.set_write_strategy(WriteStrategy::TwoPhase)
.create_mmapped(tmpfile.path())
.unwrap()
};
let table = RedbBenchDatabase::new(&db);
benchmark(table)
};

let lmdb_results = {
let tmpfile: TempDir = tempfile::tempdir_in(current_dir().unwrap()).unwrap();
let env = lmdb::Environment::new().open(tmpfile.path()).unwrap();
Expand Down Expand Up @@ -143,8 +119,6 @@ fn main() {
for results in [
redb_latency_results,
redb_throughput_results,
redb_mmap_latency_results,
redb_mmap_throughput_results,
lmdb_results,
rocksdb_results,
sled_results,
Expand All @@ -156,16 +130,7 @@ fn main() {

let mut table = comfy_table::Table::new();
table.set_width(100);
table.set_header([
"",
"redb (1PC+C)",
"redb (2PC)",
"redb (mmap-1PC+C)",
"redb (mmap-2PC)",
"lmdb",
"rocksdb",
"sled",
]);
table.set_header(["", "redb (1PC+C)", "redb (2PC)", "lmdb", "rocksdb", "sled"]);
for row in rows {
table.add_row(row);
}
Expand Down
28 changes: 0 additions & 28 deletions benches/lmdb_benchmark.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,30 +302,6 @@ fn main() {
benchmark(table)
};

let redb_mmap_latency_results = {
let tmpfile: NamedTempFile = NamedTempFile::new_in(&tmpdir).unwrap();
let db = unsafe {
redb::Database::builder()
.set_write_strategy(WriteStrategy::Checksum)
.create_mmapped(tmpfile.path())
.unwrap()
};
let table = RedbBenchDatabase::new(&db);
benchmark(table)
};

let redb_mmap_throughput_results = {
let tmpfile: NamedTempFile = NamedTempFile::new_in(&tmpdir).unwrap();
let db = unsafe {
redb::Database::builder()
.set_write_strategy(WriteStrategy::TwoPhase)
.create_mmapped(tmpfile.path())
.unwrap()
};
let table = RedbBenchDatabase::new(&db);
benchmark(table)
};

let lmdb_results = {
let tmpfile: TempDir = tempfile::tempdir_in(&tmpdir).unwrap();
let env = lmdb::Environment::new().open(tmpfile.path()).unwrap();
Expand Down Expand Up @@ -367,8 +343,6 @@ fn main() {
for results in [
redb_latency_results,
redb_throughput_results,
redb_mmap_latency_results,
redb_mmap_throughput_results,
lmdb_results,
rocksdb_results,
sled_results,
Expand All @@ -385,8 +359,6 @@ fn main() {
"",
"redb (1PC+C)",
"redb (2PC)",
"redb (mmap-1PC+C)",
"redb (mmap-2PC)",
"lmdb",
"rocksdb",
"sled",
Expand Down
4 changes: 2 additions & 2 deletions benches/multithreaded_insert_benchmark.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ fn main() {

{
let tmpfile: NamedTempFile = NamedTempFile::new_in(current_dir().unwrap()).unwrap();
let db = unsafe { Database::builder().create_mmapped(tmpfile.path()).unwrap() };
let db = Database::builder().create(tmpfile.path()).unwrap();

let start = Instant::now();
let write_txn = db.begin_write().unwrap();
Expand Down Expand Up @@ -63,7 +63,7 @@ fn main() {

{
let tmpfile: NamedTempFile = NamedTempFile::new_in(current_dir().unwrap()).unwrap();
let db = unsafe { Database::builder().create_mmapped(tmpfile.path()).unwrap() };
let db = Database::builder().create(tmpfile.path()).unwrap();

let start = Instant::now();
let write_txn = db.begin_write().unwrap();
Expand Down
8 changes: 0 additions & 8 deletions docs/design.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -371,14 +371,6 @@ the database to a savepoint the root of the b-tree is restored, and the snapshot
is diff'ed against the currently allocated pages to determine which have been allocated since the savepoint
was created. These pages are then queued to be freed, completing the rollback.

# File backends

redb has two "backends" for interacting with the file on disk. The first, the "mmap backend", is
used via the `Builder::create_mmapped()` API. This is an unsafe API because of the semantics of `mmap()`. However,
it can be faster for some use cases and it allows the OS to manage all caching of redb data.
The second backend, the "userspace backend", is entirely safe and has a user configurable cache
which it uses to cache ranges of the underlying `File`. This backend is used via the `Builder::create()` API.

# Assumptions about underlying media
redb is designed to be safe even in the event of power failure or on poorly behaved media.
Therefore, we make only a few assumptions about the guarantees provided by the underlying filesystem:
Expand Down
1 change: 0 additions & 1 deletion fuzz/fuzz_targets/common.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,6 @@ pub(crate) struct FuzzTransaction {
pub(crate) struct FuzzConfig {
pub use_checksums: bool,
pub multimap_table: bool,
pub use_mmap: bool,
pub read_cache_size: BoundedUSize<MAX_CACHE_SIZE>,
pub write_cache_size: BoundedUSize<MAX_CACHE_SIZE>,
pub thread0_transactions: Vec<FuzzTransaction>,
Expand Down
13 changes: 2 additions & 11 deletions fuzz/fuzz_targets/fuzz_redb.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,21 +284,12 @@ fuzz_target!(|config: FuzzConfig| {
} else {
WriteStrategy::TwoPhase
};
let db = if config.use_mmap {
unsafe {
Database::builder()
.set_write_strategy(write_strategy)
.set_page_size(config.page_size.value)
.create_mmapped(redb_file.path())
}
} else {
Database::builder()
let db = Database::builder()
.set_write_strategy(write_strategy)
.set_page_size(config.page_size.value)
.set_read_cache_size(config.read_cache_size.value)
.set_write_cache_size(config.write_cache_size.value)
.create(redb_file.path())
};
.create(redb_file.path());

let db = Arc::new(db.unwrap());

Expand Down
68 changes: 0 additions & 68 deletions src/db.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,6 @@ impl Database {
#[allow(clippy::too_many_arguments)]
fn new(
file: File,
use_mmap: bool,
page_size: usize,
region_size: Option<usize>,
initial_size: Option<u64>,
Expand All @@ -261,7 +260,6 @@ impl Database {
info!("Opening database {:?}", &file_path);
let mut mem = TransactionalMemory::new(
file,
use_mmap,
page_size,
region_size,
initial_size,
Expand Down Expand Up @@ -544,16 +542,12 @@ impl Builder {
}

/// Set the amount of memory (in bytes) used for caching data that has been read
///
/// This setting is ignored when calling `create_mmapped()`/`open_mmapped()`
pub fn set_read_cache_size(&mut self, bytes: usize) -> &mut Self {
self.read_cache_size_bytes = bytes;
self
}

/// Set the amount of memory (in bytes) used for caching data that has been written
///
/// This setting is ignored when calling `create_mmapped()`/`open_mmapped()`
pub fn set_write_cache_size(&mut self, bytes: usize) -> &mut Self {
self.write_cache_size_bytes = bytes;
self
Expand Down Expand Up @@ -589,38 +583,6 @@ impl Builder {

Database::new(
file,
false,
self.page_size,
self.region_size,
self.initial_size,
self.read_cache_size_bytes,
self.write_cache_size_bytes,
self.write_strategy,
)
}

/// Opens the specified file as a redb database using the mmap backend.
/// * if the file does not exist, or is an empty file, a new database will be initialized in it
/// * if the file is a valid redb database, it will be opened
/// * otherwise this function will return an error
///
/// # Safety
///
/// Caller must ensure that the memory representing the memory-mapped file is not modified externally.
/// In particular:
/// 1) the file referenced by `path` must not be concurrently modified by any other process
/// 2) an I/O failure writing back to disk must not mutate the the memory. You should consider
/// reading this paper before assuming that your OS provides this gaurantee: <https://research.cs.wisc.edu/adsl/Publications/cuttlefs-tos21.pdf>
pub unsafe fn create_mmapped(&self, path: impl AsRef<Path>) -> Result<Database> {
let file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.open(path)?;

Database::new(
file,
true,
self.page_size,
self.region_size,
self.initial_size,
Expand All @@ -638,36 +600,6 @@ impl Builder {
let file = OpenOptions::new().read(true).write(true).open(path)?;
Database::new(
file,
false,
self.page_size,
None,
self.initial_size,
self.read_cache_size_bytes,
self.write_cache_size_bytes,
None,
)
} else {
Err(Error::Io(io::Error::from(ErrorKind::InvalidData)))
}
}

/// Opens an existing redb database using the mmap backend.
///
/// # Safety
///
/// Caller must ensure that the memory representing the memory-mapped file is not modified externally.
/// In particular:
/// 1) the file referenced by `path` must not be concurrently modified by any other process
/// 2) an I/O failure writing back to disk must not mutate the the memory. You should consider
/// reading this paper before assuming that your OS provides this gaurantee: <https://research.cs.wisc.edu/adsl/Publications/cuttlefs-tos21.pdf>
pub unsafe fn open_mmapped(&self, path: impl AsRef<Path>) -> Result<Database> {
if !path.as_ref().exists() {
Err(Error::Io(ErrorKind::NotFound.into()))
} else if File::open(path.as_ref())?.metadata()?.len() > 0 {
let file = OpenOptions::new().read(true).write(true).open(path)?;
Database::new(
file,
true,
self.page_size,
None,
self.initial_size,
Expand Down
Loading