From 0462ad8f0e5bff1cfa8a803f0727b1893ebafd1a Mon Sep 17 00:00:00 2001 From: HaoyuHuang Date: Fri, 26 Jun 2020 16:32:51 -0700 Subject: [PATCH] Clean code --- AUTHORS | 12 - CMakeLists.txt | 6 +- CONTRIBUTING.md | 36 --- NEWS | 17 -- TODO | 14 - benchmarks/rdma_write_client.h | 4 +- benchmarks/rdma_write_server_worker.h | 4 +- build.sh | 7 - common/nova_common.h | 102 ++------ common/nova_config.h | 3 +- common/nova_mem_manager.h | 2 +- db/db_impl.cc | 16 +- db/lookup_index.h | 5 +- db/range_index.h | 2 - db/subrange.cpp | 1 + db/subrange_manager.h | 1 + db/version_set.h | 3 + include/leveldb/db.h | 4 +- log/log_recovery.h | 1 + log/logc_log_writer.h | 1 + log/stoc_log_manager.h | 1 + ltc/source_migration.h | 2 +- ltc/stoc_client_impl.cpp | 2 +- ltc/stoc_client_impl.h | 8 +- ltc/stoc_file_client_impl.cpp | 18 +- ltc/stoc_file_client_impl.h | 2 + ltc/storage_selector.cpp | 2 +- ltc/storage_selector.h | 2 +- ...{conn_worker.cpp => client_req_worker.cpp} | 32 +-- .../{conn_worker.h => client_req_worker.h} | 16 +- novalsm/nic_server.cpp | 17 +- novalsm/nic_server.h | 6 +- novalsm/nova_server_main.cpp | 241 ++++++++++-------- novalsm/nova_subrange_sim_test.cpp | 7 +- novalsm/rdma_admission_ctrl.h | 4 +- novalsm/rdma_msg_handler.h | 12 +- novalsm/rdma_server.cpp | 24 +- novalsm/rdma_server.h | 10 +- rdma/nova_msg_callback.h | 21 -- rdma/nova_rdma_broker.h | 4 +- rdma/nova_rdma_rc_broker.h | 11 +- rdma/rdma_msg_callback.h | 23 ++ stoc/persistent_stoc_file.h | 1 + stoc/storage_worker.h | 1 + 44 files changed, 295 insertions(+), 413 deletions(-) delete mode 100644 AUTHORS delete mode 100644 CONTRIBUTING.md delete mode 100644 NEWS delete mode 100644 TODO delete mode 100644 build.sh rename novalsm/{conn_worker.cpp => client_req_worker.cpp} (96%) rename novalsm/{conn_worker.h => client_req_worker.h} (94%) delete mode 100644 rdma/nova_msg_callback.h create mode 100644 rdma/rdma_msg_callback.h diff --git a/AUTHORS b/AUTHORS deleted file mode 100644 index 2439d7a..0000000 --- a/AUTHORS +++ /dev/null @@ -1,12 +0,0 @@ -# Names should be added to this file like so: -# Name or Organization - -Google Inc. - -# Initial version authors: -Jeffrey Dean -Sanjay Ghemawat - -# Partial list of contributors: -Kevin Regan -Johan Bilien diff --git a/CMakeLists.txt b/CMakeLists.txt index b4d773d..3b27055 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -225,11 +225,11 @@ target_sources(leveldb rdma/nova_msg_parser.h novalsm/nic_server.cpp novalsm/nic_server.h - novalsm/conn_worker.cpp - novalsm/conn_worker.h + novalsm/client_req_worker.cpp + novalsm/client_req_worker.h common/nova_common.cpp common/nova_common.h - rdma/nova_msg_callback.h + rdma/rdma_msg_callback.h rdma/nova_rdma_rc_broker.cpp rdma/nova_rdma_rc_broker.h rdma/nova_rdma_broker.h diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index a74572a..0000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,36 +0,0 @@ -# Contributing - -We'd love to accept your code patches! However, before we can take them, we -have to jump a couple of legal hurdles. - -## Contributor License Agreements - -Please fill out either the individual or corporate Contributor License -Agreement as appropriate. - -* If you are an individual writing original source code and you're sure you -own the intellectual property, then sign an [individual CLA](https://developers.google.com/open-source/cla/individual). -* If you work for a company that wants to allow you to contribute your work, -then sign a [corporate CLA](https://developers.google.com/open-source/cla/corporate). - -Follow either of the two links above to access the appropriate CLA and -instructions for how to sign and return it. - -## Submitting a Patch - -1. Sign the contributors license agreement above. -2. Decide which code you want to submit. A submission should be a set of changes -that addresses one issue in the [issue tracker](https://github.com/google/leveldb/issues). -Please don't mix more than one logical change per submission, because it makes -the history hard to follow. If you want to make a change -(e.g. add a sample or feature) that doesn't have a corresponding issue in the -issue tracker, please create one. -3. **Submitting**: When you are ready to submit, send us a Pull Request. Be -sure to include the issue number you fixed and the name you used to sign -the CLA. - -## Writing Code ## - -If your contribution contains code, please make sure that it follows -[the style guide](http://google.github.io/styleguide/cppguide.html). -Otherwise we will have to ask you to make changes, and that's no fun for anyone. diff --git a/NEWS b/NEWS deleted file mode 100644 index 3fd9924..0000000 --- a/NEWS +++ /dev/null @@ -1,17 +0,0 @@ -Release 1.2 2011-05-16 ----------------------- - -Fixes for larger databases (tested up to one billion 100-byte entries, -i.e., ~100GB). - -(1) Place hard limit on number of level-0 files. This fixes errors -of the form "too many open files". - -(2) Fixed memtable management. Before the fix, a heavy write burst -could cause unbounded memory usage. - -A fix for a logging bug where the reader would incorrectly complain -about corruption. - -Allow public access to WriteBatch contents so that users can easily -wrap a DB. diff --git a/TODO b/TODO deleted file mode 100644 index e603c07..0000000 --- a/TODO +++ /dev/null @@ -1,14 +0,0 @@ -ss -- Stats - -db -- Maybe implement DB::BulkDeleteForRange(start_key, end_key) - that would blow away files whose ranges are entirely contained - within [start_key..end_key]? For Chrome, deletion of obsolete - object stores, etc. can be done in the background anyway, so - probably not that important. -- There have been requests for MultiGet. - -After a range is completely deleted, what gets rid of the -corresponding files if we do no future changes to that range. Make -the conditions for triggering compactions fire in more situations? diff --git a/benchmarks/rdma_write_client.h b/benchmarks/rdma_write_client.h index 8aedd6b..1546853 100644 --- a/benchmarks/rdma_write_client.h +++ b/benchmarks/rdma_write_client.h @@ -7,12 +7,12 @@ #ifndef LEVELDB_RDMA_WRITE_CLIENT_H #define LEVELDB_RDMA_WRITE_CLIENT_H -#include "rdma/nova_msg_callback.h" +#include "rdma/rdma_msg_callback.h" #include "rdma/nova_rdma_broker.h" #include "common/nova_mem_manager.h" namespace nova { - class RDMAWRITEClient : public NovaMsgCallback { + class RDMAWRITEClient : public RDMAMsgCallback { public: RDMAWRITEClient(uint32_t write_size_kb, uint32_t my_server_id); diff --git a/benchmarks/rdma_write_server_worker.h b/benchmarks/rdma_write_server_worker.h index 64dd1dd..ec4fdb2 100644 --- a/benchmarks/rdma_write_server_worker.h +++ b/benchmarks/rdma_write_server_worker.h @@ -9,7 +9,7 @@ #include -#include "rdma/nova_msg_callback.h" +#include "rdma/rdma_msg_callback.h" #include "rdma/nova_rdma_broker.h" #include "stoc/persistent_stoc_file.h" #include "common/nova_mem_manager.h" @@ -34,7 +34,7 @@ namespace nova { class RDMAWRITEDiskWorker; - class RDMAWRITEServerWorker : public NovaMsgCallback { + class RDMAWRITEServerWorker : public RDMAMsgCallback { public: RDMAWRITEServerWorker(uint32_t max_run_time, uint32_t write_size_kb, bool is_local_disk_bench, bool eval_disk_horizontal_scalability, uint32_t server_id); diff --git a/build.sh b/build.sh deleted file mode 100644 index 3eb05ec..0000000 --- a/build.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -scp nova_server_main haoyu@node-0.Nova2.bg-PG0.apt.emulab.net:~/nova_debug -#scp nova_shared_main haoyu@node-1.Nova.bg-PG0.apt.emulab.net:~/nova_pool & -#scp nova_shared_main haoyu@node-2.Nova.bg-PG0.apt.emulab.net:~/nova_pool & -#scp nova_shared_main haoyu@node-3.Nova.bg-PG0.apt.emulab.net:~/nova_pool & -#scp nova_shared_main haoyu@node-4.Nova.bg-PG0.apt.emulab.net:~/nova_pool & -sleep 1000 diff --git a/common/nova_common.h b/common/nova_common.h index 0cc3a0f..aac71ab 100644 --- a/common/nova_common.h +++ b/common/nova_common.h @@ -205,30 +205,30 @@ namespace nova { /* Initial power multiplier for the hash table */ #define HASHPOWER_DEFAULT 16 - class Semaphore { - public: - Semaphore(int count_ = 0) - : count(count_) {} - - inline void notify() { - std::unique_lock lock(mtx); - count++; - cv.notify_all(); - } - - inline void wait() { - std::unique_lock lock(mtx); - while (count == 0) { - cv.wait(lock); - } - count--; - } - - private: - std::mutex mtx; - std::condition_variable cv; - int count = 0; - }; +// class Semaphore { +// public: +// Semaphore(int count_ = 0) +// : count(count_) {} +// +// inline void notify() { +// std::unique_lock lock(mtx); +// count++; +// cv.notify_all(); +// } +// +// inline void wait() { +// std::unique_lock lock(mtx); +// while (count == 0) { +// cv.wait(lock); +// } +// count--; +// } +// +// private: +// std::mutex mtx; +// std::condition_variable cv; +// int count = 0; +// }; uint32_t fastrand(); @@ -431,47 +431,6 @@ namespace nova { } }; - enum LeaseType : uint8_t { - ILEASE = 1, - QLEASE = 2 - }; - - struct LeaseEntry { - uint8_t type = 0; - // worker_id | lease_id. - uint64_t lease_id = 0; - // client_id | session_id. - uint64_t session_id = 0; - - bool empty() { - return lease_id == 0; - } - - static uint64_t size() { - return sizeof(uint8_t) + sizeof(uint64_t) + sizeof(uint64_t); - } - - static LeaseEntry chars_to_leaseitem(char *buf) { - LeaseEntry it{}; - char *tmp = buf; - memcpy(&it.type, tmp, sizeof(uint8_t)); - tmp += sizeof(uint8_t); - memcpy(&it.lease_id, tmp, sizeof(uint64_t)); - tmp += sizeof(uint64_t); - memcpy(&it.session_id, tmp, sizeof(uint64_t)); - return it; - } - - static void leaseitem_to_chars(char *_base, const LeaseEntry &entry) { - char *buf = _base; - memcpy(buf, &entry.type, sizeof(uint8_t)); - buf += sizeof(uint8_t); - memcpy(buf, &entry.lease_id, sizeof(uint64_t)); - buf += sizeof(uint64_t); - memcpy(buf, &entry.session_id, sizeof(uint64_t)); - } - }; - // Memory layout: // stale, refs, nkey, key, nval (string representation + 1), 'h', val, checksum. // data points to the beginning of the backing array. @@ -627,19 +586,6 @@ namespace nova { } }; -// inline uint32_t -// LogRecordsSize(const leveldb::LevelDBLogRecord &log_record); -// -// inline uint32_t -// LogRecordsSize(const std::vector &log_records); -// -// inline uint32_t -// EncodeLogRecord(char *buf, -// const leveldb::LevelDBLogRecord &log_record); -// -// inline uint32_t DecodeLogRecord(char *buf, -// leveldb::LevelDBLogRecord *log_record); - inline uint32_t LogRecordSize(const leveldb::LevelDBLogRecord &record) { uint32_t size = 0; diff --git a/common/nova_config.h b/common/nova_config.h index 0c08b48..88e4897 100644 --- a/common/nova_config.h +++ b/common/nova_config.h @@ -146,7 +146,7 @@ namespace nova { uint64_t log_buf_size; uint64_t max_stoc_file_size; uint64_t sstable_size; - std::string stoc_file_path; + std::string stoc_files_path; bool use_local_disk; bool enable_subrange; @@ -185,7 +185,6 @@ namespace nova { int level; int block_cache_mb; - int row_cache_mb; bool enable_lookup_index; bool enable_range_index; uint32_t num_memtables; diff --git a/common/nova_mem_manager.h b/common/nova_mem_manager.h index b8d73cd..4b2a2c8 100644 --- a/common/nova_mem_manager.h +++ b/common/nova_mem_manager.h @@ -2,7 +2,7 @@ // // Created by Haoyu Huang on 4/8/19. // Copyright (c) 2019 University of Southern California. All rights reserved. -// +// A slabclass memory manager. #ifndef NOVA_MEM_MANAGER_H #define NOVA_MEM_MANAGER_H diff --git a/db/db_impl.cc b/db/db_impl.cc index 06120be..7a6fa50 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -2634,15 +2634,15 @@ namespace leveldb { uint32_t memtable_id) { if (nova::NovaConfig::config->log_record_mode == nova::NovaLogRecordMode::LOG_RDMA && !options.local_write) { - auto dc = reinterpret_cast(options.stoc_client); - NOVA_ASSERT(dc); - dc->set_dbid(dbid_); + auto stoc = reinterpret_cast(options.stoc_client); + NOVA_ASSERT(stoc); + stoc->set_dbid(dbid_); options.stoc_client->InitiateReplicateLogRecords( nova::LogFileName(server_id_, dbid_, memtable_id), options.thread_id, dbid_, memtable_id, options.rdma_backing_mem, log_records, options.replicate_log_record_states); - dc->Wait(); + stoc->Wait(); } } @@ -2652,9 +2652,9 @@ namespace leveldb { uint32_t memtable_id) { if (nova::NovaConfig::config->log_record_mode == nova::NovaLogRecordMode::LOG_RDMA && !options.local_write) { - auto dc = reinterpret_cast(options.stoc_client); - NOVA_ASSERT(dc); - dc->set_dbid(dbid_); + auto stoc = reinterpret_cast(options.stoc_client); + NOVA_ASSERT(stoc); + stoc->set_dbid(dbid_); LevelDBLogRecord log_record = {}; log_record.sequence_number = last_sequence; log_record.key = key; @@ -2666,7 +2666,7 @@ namespace leveldb { options.thread_id, dbid_, memtable_id, options.rdma_backing_mem, {log_record}, options.replicate_log_record_states); - dc->Wait(); + stoc->Wait(); } } diff --git a/db/lookup_index.h b/db/lookup_index.h index f335ba3..77090a1 100644 --- a/db/lookup_index.h +++ b/db/lookup_index.h @@ -2,7 +2,8 @@ // // Created by Haoyu Huang on 5/19/20. // Copyright (c) 2020 University of Southern California. All rights reserved. -// +// It assumes a fixed size lookup index. One may swap it with any implementation of concurrent hash map. +// TODO: Support cleaning up the lookup index to remove entries that point to obsolete memtable ids. #ifndef LEVELDB_LOOKUP_INDEX_H #define LEVELDB_LOOKUP_INDEX_H @@ -29,6 +30,7 @@ namespace leveldb { uint32_t Encode(char *buf); void Decode(Slice *buf); + private: uint32_t size_ = 0; TableLocation *table_locator_ = nullptr; @@ -36,5 +38,4 @@ namespace leveldb { } - #endif //LEVELDB_LOOKUP_INDEX_H diff --git a/db/range_index.h b/db/range_index.h index 81ece7c..0a1a388 100644 --- a/db/range_index.h +++ b/db/range_index.h @@ -15,8 +15,6 @@ #include "leveldb/slice.h" #include "version_set.h" -#define MAX_LIVE_VERSIONS 100000 - namespace leveldb { struct RangeTables { std::set memtable_ids; diff --git a/db/subrange.cpp b/db/subrange.cpp index 48f11bc..b1c89c5 100644 --- a/db/subrange.cpp +++ b/db/subrange.cpp @@ -2,6 +2,7 @@ // // Created by Haoyu Huang on 5/4/20. // Copyright (c) 2020 University of Southern California. All rights reserved. +// It assumes keys are integers. // #include "leveldb/subrange.h" diff --git a/db/subrange_manager.h b/db/subrange_manager.h index 0c7a2cc..aae1a28 100644 --- a/db/subrange_manager.h +++ b/db/subrange_manager.h @@ -2,6 +2,7 @@ // // Created by Haoyu Huang on 5/4/20. // Copyright (c) 2020 University of Southern California. All rights reserved. +// It assumes keys are integers. // #ifndef LEVELDB_SUBRANGE_MANAGER_H diff --git a/db/version_set.h b/db/version_set.h index 91da276..8d97b54 100644 --- a/db/version_set.h +++ b/db/version_set.h @@ -29,6 +29,8 @@ #include "table_cache.h" #include "range_index.h" +// Maintain this many live memtables. +// The program exits when the number of memtables exceeds this threshold. #define MAX_LIVE_MEMTABLES 100000 namespace leveldb { @@ -216,6 +218,7 @@ namespace leveldb { uint32_t Encode(char *buf); void Decode(Slice *buf); + int refs_ = 0; // Number of live refs to this version private: diff --git a/include/leveldb/db.h b/include/leveldb/db.h index 25a08f2..7573654 100644 --- a/include/leveldb/db.h +++ b/include/leveldb/db.h @@ -13,7 +13,7 @@ #include "leveldb/export.h" #include "leveldb/iterator.h" #include "leveldb/options.h" -#include "rdma/nova_msg_callback.h" +#include "rdma/rdma_msg_callback.h" namespace leveldb { @@ -210,7 +210,7 @@ namespace leveldb { virtual void StartCompaction() = 0; std::vector dbs_; - std::vector rdma_threads_; + std::vector rdma_threads_; uint64_t number_of_memtable_hits_ = 0; uint64_t number_of_gets_ = 0; diff --git a/log/log_recovery.h b/log/log_recovery.h index d80ca83..71cf48a 100644 --- a/log/log_recovery.h +++ b/log/log_recovery.h @@ -15,6 +15,7 @@ namespace leveldb { std::string logfile; }; + // TODO: For refactoring. class LogRecovery { public: void Recover(std::vector memtables_to_recover); diff --git a/log/logc_log_writer.h b/log/logc_log_writer.h index 5f5e55e..a411d05 100644 --- a/log/logc_log_writer.h +++ b/log/logc_log_writer.h @@ -16,6 +16,7 @@ #include "novalsm/rdma_admission_ctrl.h" namespace leveldb { + // Replicate log records across StoCs. class LogCLogWriter { public: LogCLogWriter(nova::NovaRDMABroker *rdma_broker, diff --git a/log/stoc_log_manager.h b/log/stoc_log_manager.h index a94b45c..6f36dc9 100644 --- a/log/stoc_log_manager.h +++ b/log/stoc_log_manager.h @@ -14,6 +14,7 @@ #include "leveldb/log_writer.h" namespace nova { + // Manage in-memory log files to provide high availability. class StoCInMemoryLogFileManager { public: StoCInMemoryLogFileManager(NovaMemManager *mem_manager); diff --git a/ltc/source_migration.h b/ltc/source_migration.h index 6b17a70..613097a 100644 --- a/ltc/source_migration.h +++ b/ltc/source_migration.h @@ -2,7 +2,7 @@ // // Created by Haoyu Huang on 6/18/20. // Copyright (c) 2020 University of Southern California. All rights reserved. -// +// TODO: Support migration. #ifndef LEVELDB_SOURCE_MIGRATION_H #define LEVELDB_SOURCE_MIGRATION_H diff --git a/ltc/stoc_client_impl.cpp b/ltc/stoc_client_impl.cpp index 45f1331..e2ffd99 100644 --- a/ltc/stoc_client_impl.cpp +++ b/ltc/stoc_client_impl.cpp @@ -222,7 +222,7 @@ namespace leveldb { IncrementReqId(); NOVA_LOG(DEBUG) << fmt::format( - "dcclient[{}]: Read log file: dc:{} off:{} size:{} req:{}", + "stoc-client[{}]: Read log file: stoc:{} off:{} size:{} req:{}", stoc_client_id_, stoc_id, remote_offset, size, req_id); return req_id; diff --git a/ltc/stoc_client_impl.h b/ltc/stoc_client_impl.h index e9742ee..923d2b0 100644 --- a/ltc/stoc_client_impl.h +++ b/ltc/stoc_client_impl.h @@ -2,7 +2,8 @@ // // Created by Haoyu Huang on 1/8/20. // Copyright (c) 2020 University of Southern California. All rights reserved. -// +// StoCBlockingClient uses a semaphore to wait for the response after issuing a request. +// StoCRDMAClient is based on RDMA. #ifndef LEVELDB_STOC_CLIENT_IMPL_H #define LEVELDB_STOC_CLIENT_IMPL_H @@ -133,12 +134,13 @@ namespace leveldb { class StoCRDMAClient : public StoCClient { public: - StoCRDMAClient(uint32_t dc_client_id, nova::NovaRDMABroker *rdma_broker, + StoCRDMAClient(uint32_t stoc_client_id, + nova::NovaRDMABroker *rdma_broker, nova::NovaMemManager *mem_manager, leveldb::LogCLogWriter *rdma_log_writer, uint32_t lower_req_id, uint32_t upper_req_id, RDMAServer *rdma_server) - : stoc_client_id_(dc_client_id), rdma_broker_(rdma_broker), + : stoc_client_id_(stoc_client_id), rdma_broker_(rdma_broker), mem_manager_(mem_manager), rdma_log_writer_(rdma_log_writer), lower_req_id_(lower_req_id), upper_req_id_(upper_req_id), diff --git a/ltc/stoc_file_client_impl.cpp b/ltc/stoc_file_client_impl.cpp index e82e109..29506ed 100644 --- a/ltc/stoc_file_client_impl.cpp +++ b/ltc/stoc_file_client_impl.cpp @@ -577,20 +577,19 @@ namespace leveldb { backing_mem_block = scratch; } NOVA_ASSERT(backing_mem_block); - auto dc = reinterpret_cast(read_options.stoc_client); - dc->set_dbid(dbid_); - uint32_t req_id = dc->InitiateReadDataBlock( + auto stoc_client = reinterpret_cast(read_options.stoc_client); + stoc_client->set_dbid(dbid_); + uint32_t req_id = stoc_client->InitiateReadDataBlock( block_handle, offset, n, backing_mem_block, n, "", true); NOVA_LOG(rdmaio::DEBUG) << fmt::format("t[{}]: CCRead req:{} start db:{} fn:{} s:{}", read_options.thread_id, req_id, dbid_, file_number_, n); - dc->Wait(); + stoc_client->Wait(); NOVA_LOG(rdmaio::DEBUG) << fmt::format("t[{}]: CCRead req:{} complete db:{} fn:{} s:{}", read_options.thread_id, req_id, dbid_, file_number_, n); -// NOVA_ASSERT(dc->IsDone(req_id, nullptr, nullptr)); NOVA_ASSERT(nova::IsRDMAWRITEComplete(backing_mem_block, n)) << fmt::format("t[{}]: {}", read_options.thread_id, req_id); ptr = backing_mem_block; @@ -634,14 +633,14 @@ namespace leveldb { NOVA_ASSERT(backing_mem_table_) << "Running out of memory"; uint64_t offset = 0; uint32_t reqs[meta_->data_block_group_handles.size()]; - auto dc = reinterpret_cast(stoc_client); - dc->set_dbid(dbid_); + auto stoc_block_client = reinterpret_cast(stoc_client); + stoc_block_client->set_dbid(dbid_); for (int i = 0; i < meta_->data_block_group_handles.size(); i++) { const StoCBlockHandle &handle = meta_->data_block_group_handles[i]; NOVA_ASSERT(offset + handle.size <= meta_->file_size); uint64_t id = (((uint64_t) handle.server_id) << 32) | handle.stoc_file_id; - reqs[i] = dc->InitiateReadDataBlock(handle, + reqs[i] = stoc_block_client->InitiateReadDataBlock(handle, handle.offset, handle.size, backing_mem_table_ + @@ -657,12 +656,11 @@ namespace leveldb { } // Wait for all reads to complete. for (int i = 0; i < meta_->data_block_group_handles.size(); i++) { - dc->Wait(); + stoc_block_client->Wait(); } offset = 0; for (int i = 0; i < meta_->data_block_group_handles.size(); i++) { const StoCBlockHandle &handle = meta_->data_block_group_handles[i]; -// NOVA_ASSERT(dc->IsDone(reqs[i], nullptr, nullptr)); NOVA_ASSERT(nova::IsRDMAWRITEComplete(backing_mem_table_ + offset, handle.size)); offset += handle.size; diff --git a/ltc/stoc_file_client_impl.h b/ltc/stoc_file_client_impl.h index 4ca031c..5b50280 100644 --- a/ltc/stoc_file_client_impl.h +++ b/ltc/stoc_file_client_impl.h @@ -16,6 +16,8 @@ namespace leveldb { + // A stoc writable file client is implemented based on memfile. + // It first writes data to its memory and then RDMA WRITEs to StoCs. class StoCWritableFileClient : public MemFile { public: StoCWritableFileClient(Env *env, diff --git a/ltc/storage_selector.cpp b/ltc/storage_selector.cpp index 166a2f1..a3a35fd 100644 --- a/ltc/storage_selector.cpp +++ b/ltc/storage_selector.cpp @@ -85,7 +85,7 @@ namespace leveldb { storage_stats[i].response, nullptr)); } - // sort the dc stats. + // sort the stoc stats. std::sort(storage_stats.begin(), storage_stats.end(), stoc_stats_comparator); for (int i = 0; i < num_storage_to_select; i++) { diff --git a/ltc/storage_selector.h b/ltc/storage_selector.h index 4d4d692..933b2a8 100644 --- a/ltc/storage_selector.h +++ b/ltc/storage_selector.h @@ -2,7 +2,7 @@ // // Created by Haoyu Huang on 5/13/20. // Copyright (c) 2020 University of Southern California. All rights reserved. -// +// Select StoCs to store data based on scatter policy. #ifndef LEVELDB_STORAGE_SELECTOR_H #define LEVELDB_STORAGE_SELECTOR_H diff --git a/novalsm/conn_worker.cpp b/novalsm/client_req_worker.cpp similarity index 96% rename from novalsm/conn_worker.cpp rename to novalsm/client_req_worker.cpp index 8122422..9dc6b0f 100644 --- a/novalsm/conn_worker.cpp +++ b/novalsm/client_req_worker.cpp @@ -4,7 +4,7 @@ // Copyright (c) 2019 University of Southern California. All rights reserved. // -#include "conn_worker.h" +#include "client_req_worker.h" #include "common/nova_console_logging.h" #include "common/nova_common.h" @@ -41,7 +41,7 @@ namespace nova { SocketState socket_write_handler(int fd, Connection *conn) { NOVA_ASSERT(conn->response_size < NovaConfig::config->max_msg_size); - NICConnWorker *store = (NICConnWorker *) conn->worker; + NICClientReqWorker *store = (NICClientReqWorker *) conn->worker; struct iovec iovec_array[1]; iovec_array[0].iov_base = conn->response_buf + conn->response_ind; iovec_array[0].iov_len = conn->response_size - conn->response_ind; @@ -85,7 +85,7 @@ namespace nova { NOVA_ASSERT(fd == conn->fd) << fd << ":" << conn->fd; SocketState state; - NICConnWorker *worker = (NICConnWorker *) conn->worker; + NICClientReqWorker *worker = (NICClientReqWorker *) conn->worker; if (conn->state == ConnState::READ) { if (worker->stats.nreqs % 100 == 0) { @@ -133,7 +133,7 @@ namespace nova { } void write_socket_complete(int fd, Connection *conn) { - NICConnWorker *worker = (NICConnWorker *) conn->worker; + NICClientReqWorker *worker = (NICClientReqWorker *) conn->worker; if (worker->stats.nreqs % 99 == 0 && worker->stats.nreqs > 0) { @@ -160,7 +160,7 @@ namespace nova { bool process_socket_get(int fd, Connection *conn, bool no_redirect) { // Stats. - NICConnWorker *worker = (NICConnWorker *) conn->worker; + NICClientReqWorker *worker = (NICClientReqWorker *) conn->worker; worker->stats.ngets++; char *buf = worker->request_buf; NOVA_ASSERT( @@ -207,7 +207,7 @@ namespace nova { bool process_reintialize_qps(int fd, Connection *conn) { NOVA_LOG(rdmaio::INFO) << "Reinitialize QPs"; - NICConnWorker *worker = (NICConnWorker *) conn->worker; + NICClientReqWorker *worker = (NICClientReqWorker *) conn->worker; for (int i = 0; i < worker->rdma_threads.size(); i++) { auto *thread = reinterpret_cast(worker->rdma_threads[i]); thread->should_pause = true; @@ -245,7 +245,7 @@ namespace nova { bool process_socket_stats_request(int fd, Connection *conn) { NOVA_LOG(rdmaio::INFO) << "Obtain stats"; - NICConnWorker *worker = (NICConnWorker *) conn->worker; + NICClientReqWorker *worker = (NICClientReqWorker *) conn->worker; int num_l0_sstables = 0; bool needs_compaction = false; for (auto db : worker->dbs_) { @@ -274,7 +274,7 @@ namespace nova { bool process_close_stoc_files(int fd, Connection *conn) { NOVA_LOG(rdmaio::INFO) << "Close StoC files"; - NICConnWorker *worker = (NICConnWorker *) conn->worker; + NICClientReqWorker *worker = (NICClientReqWorker *) conn->worker; for (int i = 0; i < worker->rdma_threads.size(); i++) { auto *thread = reinterpret_cast(worker->rdma_threads[i]); @@ -311,7 +311,7 @@ namespace nova { bool process_socket_scan(int fd, Connection *conn) { - NICConnWorker *worker = (NICConnWorker *) conn->worker; + NICClientReqWorker *worker = (NICClientReqWorker *) conn->worker; worker->stats.nscans++; char *buf = worker->request_buf; NOVA_ASSERT(buf[0] == RequestType::REQ_SCAN) << buf; @@ -381,7 +381,7 @@ namespace nova { bool process_socket_put(int fd, Connection *conn) { // Stats. - NICConnWorker *worker = (NICConnWorker *) conn->worker; + NICClientReqWorker *worker = (NICClientReqWorker *) conn->worker; worker->stats.nputs++; char *buf = worker->request_buf; NOVA_ASSERT(buf[0] == RequestType::PUT) << buf; @@ -458,7 +458,7 @@ namespace nova { } bool process_socket_request_handler(int fd, Connection *conn) { - auto worker = (NICConnWorker*) conn->worker; + auto worker = (NICClientReqWorker*) conn->worker; char *buf = worker->request_buf; if (buf[0] == RequestType::GET) { return process_socket_get(fd, conn, /*no_redirect=*/false); @@ -490,7 +490,7 @@ namespace nova { SocketState socket_read_handler(int fd, short which, Connection *conn) { NOVA_ASSERT((which & EV_READ) > 0) << which; - NICConnWorker *worker = (NICConnWorker *) conn->worker; + NICClientReqWorker *worker = (NICClientReqWorker *) conn->worker; char *buf = worker->request_buf + worker->req_ind; bool complete = false; @@ -535,7 +535,7 @@ namespace nova { } void stats_handler(int fd, short which, void *arg) { - NICConnWorker *store = (NICConnWorker *) arg; + NICClientReqWorker *store = (NICClientReqWorker *) arg; Stats diff = store->stats.diff(store->prev_stats); uint64_t service_time = 0; uint64_t read_service_time = 0; @@ -597,7 +597,7 @@ namespace nova { } void new_conn_handler(int fd, short which, void *arg) { - NICConnWorker *store = (NICConnWorker *) arg; + NICClientReqWorker *store = (NICClientReqWorker *) arg; new_conn_mutex.lock(); store->conn_mu.lock(); store->nconns += store->conn_queue.size(); @@ -630,7 +630,7 @@ namespace nova { new_conn_mutex.unlock(); } - void NICConnWorker::Start() { + void NICClientReqWorker::Start() { NOVA_LOG(DEBUG) << "memstore[" << thread_id_ << "]: " << "starting mem worker"; @@ -723,7 +723,7 @@ namespace nova { event_flags = new_flags; NOVA_ASSERT(event_del(&event) == 0) << fd; NOVA_ASSERT( - event_assign(&event, ((NICConnWorker *) worker)->base, fd, + event_assign(&event, ((NICClientReqWorker *) worker)->base, fd, new_flags, event_handler, this) == diff --git a/novalsm/conn_worker.h b/novalsm/client_req_worker.h similarity index 94% rename from novalsm/conn_worker.h rename to novalsm/client_req_worker.h index 439b5e2..4496450 100644 --- a/novalsm/conn_worker.h +++ b/novalsm/client_req_worker.h @@ -2,10 +2,10 @@ // // Created by Haoyu Huang on 3/28/19. // Copyright (c) 2019 University of Southern California. All rights reserved. -// +// Client request worker based on libevent. -#ifndef RLIB_NOVA_MEM_STORE_H -#define RLIB_NOVA_MEM_STORE_H +#ifndef CLIENT_REQ_WORKER_H +#define CLIENT_REQ_WORKER_H #include @@ -14,7 +14,7 @@ #include #include -#include "rdma/nova_msg_callback.h" +#include "rdma/rdma_msg_callback.h" #include "rdma/nova_rdma_broker.h" #include "common/nova_common.h" #include "common/nova_config.h" @@ -95,9 +95,9 @@ namespace nova { std::vector workers; }; - class NICConnWorker { + class NICClientReqWorker { public: - NICConnWorker(int thread_id) + NICClientReqWorker(int thread_id) : thread_id_(thread_id) { NOVA_LOG(INFO) << "memstore[" << thread_id << "]: " @@ -140,7 +140,7 @@ namespace nova { std::vector dbs_; struct event_base *base = nullptr; rdmaio::RdmaCtrl *ctrl_; - std::vector rdma_threads; + std::vector rdma_threads; leveldb::StocPersistentFileManager *stoc_file_manager_; leveldb::StoCBlockClient *stoc_client_; @@ -164,4 +164,4 @@ namespace nova { }; } -#endif //RLIB_NOVA_MEM_STORE_H +#endif //CLIENT_REQ_WORKER_H diff --git a/novalsm/nic_server.cpp b/novalsm/nic_server.cpp index a3cef98..26dd810 100644 --- a/novalsm/nic_server.cpp +++ b/novalsm/nic_server.cpp @@ -20,7 +20,7 @@ #include "ltc/db_helper.h" namespace nova { - void start(NICConnWorker *store) { + void start(NICClientReqWorker *store) { store->Start(); } @@ -342,12 +342,6 @@ namespace nova { block_cache->TotalCapacity(), NovaConfig::config->block_cache_mb); } - if (NovaConfig::config->row_cache_mb > 0) { - uint64_t row_cache_size = - (uint64_t) (NovaConfig::config->row_cache_mb) * 1024 * - 1024; - row_cache = leveldb::NewLRUCache(row_cache_size); - } leveldb::MemTablePool *pool = new leveldb::MemTablePool; pool->num_available_memtables_ = NovaConfig::config->num_memtables; @@ -362,13 +356,13 @@ namespace nova { NovaConfig::config->ltc_servers.size(); leveldb::StocPersistentFileManager *stoc_file_manager = new leveldb::StocPersistentFileManager( - env, mem_manager, NovaConfig::config->stoc_file_path, + env, mem_manager, NovaConfig::config->stoc_files_path, NovaConfig::config->max_stoc_file_size, NovaConfig::config->servers.size(), NovaConfig::config->my_server_id, nranges); - std::vector rdma_threads; + std::vector rdma_threads; std::vector db_clients; for (int db_index = 0; db_index < ndbs; db_index++) { auto reorg = new leveldb::LTCCompactionThread(mem_manager); @@ -398,6 +392,7 @@ namespace nova { for (auto &dbname : dbnames) { mkdirs(dbname.c_str()); } + // Assign request id space so that they won't conflict. int worker_id = 0; uint32_t max_req_id = UINT32_MAX - 1; uint32_t range_per_server = @@ -555,7 +550,7 @@ namespace nova { for (int i = 0; i < NovaConfig::config->num_conn_workers; i++) { - conn_workers.push_back(new NICConnWorker(i)); + conn_workers.push_back(new NICClientReqWorker(i)); conn_workers[i]->set_dbs(dbs_); conn_workers[i]->mem_manager_ = mem_manager; @@ -853,7 +848,7 @@ namespace nova { make_socket_non_blocking(client_fd); NOVA_LOG(DEBUG) << "register " << client_fd; - NICConnWorker *store = server->conn_workers[server->current_conn_worker_id_]; + NICClientReqWorker *store = server->conn_workers[server->current_conn_worker_id_]; if (NovaConfig::config->num_conn_workers == 1) { server->current_conn_worker_id_ = 0; } else { diff --git a/novalsm/nic_server.h b/novalsm/nic_server.h index 4bf8adf..0298955 100644 --- a/novalsm/nic_server.h +++ b/novalsm/nic_server.h @@ -9,7 +9,7 @@ #include "leveldb/db_types.h" #include "common/nova_mem_manager.h" -#include "conn_worker.h" +#include "client_req_worker.h" #include "common/nova_config.h" #include "rdma/nova_rdma_broker.h" #include "rdma/nova_rdma_rc_broker.h" @@ -20,7 +20,7 @@ #include "ltc/stat_thread.h" namespace nova { - class NICConnWorker; + class NICClientReqWorker; class LoadThread { public: @@ -63,7 +63,7 @@ namespace nova { NovaMemManager *mem_manager; StoCInMemoryLogFileManager *log_manager; - std::vector conn_workers; + std::vector conn_workers; std::vector fg_rdma_msg_handlers; std::vector bg_rdma_msg_handlers; diff --git a/novalsm/nova_server_main.cpp b/novalsm/nova_server_main.cpp index 00f8ee7..f63692b 100644 --- a/novalsm/nova_server_main.cpp +++ b/novalsm/nova_server_main.cpp @@ -2,7 +2,7 @@ // // Created by Haoyu Huang on 2/20/19. // Copyright (c) 2019 University of Southern California. All rights reserved. -// +// NovaLSM main class. #include "rdma/rdma_ctrl.hpp" @@ -23,12 +23,12 @@ using namespace std; using namespace rdmaio; using namespace nova; -DEFINE_string(db_path, "/tmp/rdma", "level db path"); -DEFINE_string(rtable_path, "/tmp/rtables", "RTable path"); +DEFINE_string(db_path, "/tmp/db", "level db path"); +DEFINE_string(stoc_files_path, "/tmp/stoc", "StoC files path"); -DEFINE_string(cc_servers, "localhost:11211", "A list of servers"); +DEFINE_string(all_servers, "localhost:11211", "A list of servers"); DEFINE_int64(server_id, -1, "Server id."); -DEFINE_int64(number_of_ccs, 0, "The first n are CCs and the rest are DCs."); +DEFINE_int64(number_of_ltcs, 0, "The first n are LTCs and the rest are StoCs."); DEFINE_uint64(mem_pool_size_gb, 0, "Memory pool size in GB."); DEFINE_uint64(use_fixed_value_size, 0, "Fixed value size."); @@ -38,71 +38,88 @@ DEFINE_uint64(rdma_max_msg_size, 0, "The maximum message size used by RDMA."); DEFINE_uint64(rdma_max_num_sends, 0, "The maximum number of pending RDMA sends. This includes READ/WRITE/SEND. We also post the same number of RECV events. "); DEFINE_uint64(rdma_doorbell_batch_size, 0, "The doorbell batch size."); -DEFINE_bool(enable_rdma, false, "Enable RDMA messaging."); +DEFINE_bool(enable_rdma, false, "Enable RDMA."); DEFINE_bool(enable_load_data, false, "Enable loading data."); -DEFINE_string(cc_config_path, "/tmp/uniform-3-32-10000000-frags.txt", - "The path that stores fragment configuration."); -DEFINE_uint64(cc_num_conn_workers, 0, "Number of connection threads."); -DEFINE_uint32(cc_num_async_workers, 0, "Number of async worker threads."); -DEFINE_uint32(cc_num_compaction_workers, 0, +DEFINE_string(ltc_config_path, "/tmp/uniform-3-32-10000000-frags.txt", + "The path that stores the configuration."); +DEFINE_uint64(ltc_num_client_workers, 0, "Number of client worker threads."); +DEFINE_uint32(num_rdma_fg_workers, 0, + "Number of RDMA foreground worker threads."); +DEFINE_uint32(num_compaction_workers, 0, "Number of compaction worker threads."); -DEFINE_uint32(cc_num_rdma_compaction_workers, 0, - "Number of rdma compaction worker threads."); +DEFINE_uint32(num_rdma_bg_workers, 0, + "Number of RDMA background worker threads."); -DEFINE_uint32(cc_num_storage_workers, 0, +DEFINE_uint32(num_storage_workers, 0, "Number of storage worker threads."); -DEFINE_uint32(cc_rtable_num_servers_scatter_data_blocks, 0, - "Number of servers to scatter data blocks "); - -DEFINE_uint64(cc_block_cache_mb, 0, "leveldb block cache size in mb"); -DEFINE_uint64(cc_row_cache_mb, 0, "leveldb row cache size in mb"); - -DEFINE_uint32(cc_num_memtables, 0, ""); -DEFINE_uint32(cc_num_memtable_partitions, 0, ""); -DEFINE_bool(cc_enable_table_locator, false, ""); -DEFINE_bool(cc_enable_range_index, false, ""); - -DEFINE_uint32(cc_l0_start_compaction_mb, 0, ""); -DEFINE_uint32(cc_l0_stop_write_mb, 0, ""); -DEFINE_int32(level, 2, ""); - -DEFINE_uint64(cc_write_buffer_size_mb, 0, "write buffer size in mb"); -DEFINE_uint64(cc_sstable_size_mb, 0, "sstable size in mb"); -DEFINE_uint32(cc_log_buf_size, 0, "log buffer size"); -DEFINE_uint32(cc_rtable_size_mb, 0, "RTable size"); -DEFINE_bool(cc_local_disk, false, ""); -DEFINE_string(cc_scatter_policy, "random", "random/stats"); -DEFINE_string(cc_log_record_mode, "none", "none/rdma"); -DEFINE_uint32(cc_num_log_replicas, 0, ""); -DEFINE_string(cc_memtable_type, "", "pool/static_partition"); - -DEFINE_bool(cc_recover_dbs, false, "recovery"); -DEFINE_uint32(cc_num_recovery_threads, 32, "recovery"); - -DEFINE_bool(cc_enable_subrange, false, ""); -DEFINE_bool(cc_enable_subrange_reorg, false, ""); -DEFINE_double(cc_sampling_ratio, 1, ""); -DEFINE_string(cc_zipfian_dist, "/tmp/zipfian", ""); -DEFINE_string(cc_client_access_pattern, "uniform", ""); -DEFINE_uint32(cc_num_tinyranges_per_subrange, 10, ""); - -DEFINE_bool(cc_enable_detailed_db_stats, false, ""); -DEFINE_bool(cc_enable_flush_multiple_memtables, false, ""); -DEFINE_uint32(cc_subrange_no_flush_num_keys, 100, ""); -DEFINE_string(cc_major_compaction_type, "no", "no/st/lc/sc"); -DEFINE_uint32(cc_major_compaction_max_parallism, 1, ""); -DEFINE_uint32(cc_major_compaction_max_tables_in_a_set, 15, ""); - +DEFINE_uint32(ltc_num_stocs_scatter_data_blocks, 0, + "Number of StoCs to scatter data blocks of an SSTable."); + +DEFINE_uint64(block_cache_mb, 0, "block cache size in mb"); +DEFINE_uint64(row_cache_mb, 0, "row cache size in mb. Not supported"); + +DEFINE_uint32(num_memtables, 0, "Number of memtables."); +DEFINE_uint32(num_memtable_partitions, 0, + "Number of memtable partitions. One active memtable per partition."); +DEFINE_bool(enable_lookup_index, false, "Enable lookup index."); +DEFINE_bool(enable_range_index, false, "Enable range index."); + +DEFINE_uint32(l0_start_compaction_mb, 0, + "Level-0 size to start compaction in MB."); +DEFINE_uint32(l0_stop_write_mb, 0, "Level-0 size to stall writes in MB."); +DEFINE_int32(level, 2, "Number of levels."); + +DEFINE_uint64(memtable_size_mb, 0, "memtable size in mb"); +DEFINE_uint64(sstable_size_mb, 0, "sstable size in mb"); +DEFINE_uint32(cc_log_buf_size, 0, + "log buffer size. Not supported. Same as memtable size."); +DEFINE_uint32(max_stoc_file_size_mb, 0, "Max StoC file size in MB"); +DEFINE_bool(use_local_disk, false, + "Enable LTC to write data to its local disk."); +DEFINE_string(scatter_policy, "random", + "Policy to scatter an SSTable, i.e., random/power_of_two"); +DEFINE_string(log_record_mode, "none", + "Policy for LogC to replicate log records, i.e., none/rdma"); +DEFINE_uint32(num_log_replicas, 0, "Number of replicas for a log record."); +DEFINE_string(memtable_type, "", "Memtable type, i.e., pool/static_partition"); + +DEFINE_bool(recover_dbs, false, "Enable recovery"); +DEFINE_uint32(num_recovery_threads, 32, "Number of recovery threads"); + +DEFINE_bool(enable_subrange, false, "Enable subranges"); +DEFINE_bool(enable_subrange_reorg, false, "Enable subrange reorganization."); +DEFINE_double(sampling_ratio, 1, + "Sampling ratio on memtables for subrange reorg. A value between 0 and 1."); +DEFINE_string(zipfian_dist_ref_counts, "/tmp/zipfian", + "Zipfian ref count file used to report load imbalance across subranges."); +DEFINE_string(client_access_pattern, "uniform", + "Client access pattern used to report load imbalance across subranges."); +DEFINE_uint32(num_tinyranges_per_subrange, 10, + "Number of tiny ranges per subrange."); + +DEFINE_bool(enable_detailed_db_stats, false, + "Enable detailed stats. It will report stats such as number of overlapping SSTables between Level-0 and Level-1."); +DEFINE_bool(enable_flush_multiple_memtables, false, + "Enable a compaction thread to compact mulitple memtables at the same time."); +DEFINE_uint32(subrange_no_flush_num_keys, 100, + "A subrange merges memtables into new a memtable if its contained number of unique keys is less than this threshold."); +DEFINE_string(major_compaction_type, "no", + "Major compaction type: i.e., no/lc/sc"); +DEFINE_uint32(major_compaction_max_parallism, 1, + "The maximum compaction parallelism."); +DEFINE_uint32(major_compaction_max_tables_in_a_set, 15, + "The maximum number of SSTables in a compaction job."); NovaConfig *NovaConfig::config; std::atomic_int_fast32_t leveldb::EnvBGThread::bg_flush_memtable_thread_id_seq; +std::atomic_int_fast32_t nova::StorageWorker::storage_file_number_seq; +// Sequence id to assign tasks to a thread in a round-robin manner. +std::atomic_int_fast32_t nova::RDMAServerImpl::compaction_storage_worker_seq_id_; std::atomic_int_fast32_t leveldb::EnvBGThread::bg_compaction_thread_id_seq; std::atomic_int_fast32_t nova::RDMAServerImpl::fg_storage_worker_seq_id_; std::atomic_int_fast32_t nova::RDMAServerImpl::bg_storage_worker_seq_id_; std::atomic_int_fast32_t leveldb::StoCBlockClient::rdma_worker_seq_id_; -std::atomic_int_fast32_t nova::StorageWorker::storage_file_number_seq; -std::atomic_int_fast32_t nova::RDMAServerImpl::compaction_storage_worker_seq_id_; std::unordered_map leveldb::Version::last_fnfile; NovaGlobalVariables NovaGlobalVariables::global; @@ -121,13 +138,13 @@ void StartServer() { NovaConfig::config->nnovabuf = ntotal; NOVA_ASSERT(buf != NULL) << "Not enough memory"; - if (!FLAGS_cc_recover_dbs) { + if (!FLAGS_recover_dbs) { system(fmt::format("exec rm -rf {}/*", NovaConfig::config->db_path).data()); system(fmt::format("exec rm -rf {}/*", - NovaConfig::config->stoc_file_path).data()); + NovaConfig::config->stoc_files_path).data()); } - mkdirs(NovaConfig::config->stoc_file_path.data()); + mkdirs(NovaConfig::config->stoc_files_path.data()); mkdirs(NovaConfig::config->db_path.data()); auto *mem_server = new NICServer(rdma_ctrl, buf, port); mem_server->Start(); @@ -153,7 +170,7 @@ int main(int argc, char *argv[]) { } NovaConfig::config = new NovaConfig; - NovaConfig::config->stoc_file_path = FLAGS_rtable_path; + NovaConfig::config->stoc_files_path = FLAGS_stoc_files_path; NovaConfig::config->mem_pool_size_gb = FLAGS_mem_pool_size_gb; NovaConfig::config->load_default_value_size = FLAGS_use_fixed_value_size; @@ -163,23 +180,22 @@ int main(int argc, char *argv[]) { NovaConfig::config->rdma_max_num_sends = FLAGS_rdma_max_num_sends; NovaConfig::config->rdma_doorbell_batch_size = FLAGS_rdma_doorbell_batch_size; - NovaConfig::config->block_cache_mb = FLAGS_cc_block_cache_mb; - NovaConfig::config->row_cache_mb = FLAGS_cc_row_cache_mb; - NovaConfig::config->memtable_size_mb = FLAGS_cc_write_buffer_size_mb; + NovaConfig::config->block_cache_mb = FLAGS_block_cache_mb; + NovaConfig::config->memtable_size_mb = FLAGS_memtable_size_mb; NovaConfig::config->db_path = FLAGS_db_path; NovaConfig::config->enable_rdma = FLAGS_enable_rdma; NovaConfig::config->enable_load_data = FLAGS_enable_load_data; - NovaConfig::config->major_compaction_type = FLAGS_cc_major_compaction_type; - NovaConfig::config->enable_flush_multiple_memtables = FLAGS_cc_enable_flush_multiple_memtables; - NovaConfig::config->major_compaction_max_parallism = FLAGS_cc_major_compaction_max_parallism; - NovaConfig::config->major_compaction_max_tables_in_a_set = FLAGS_cc_major_compaction_max_tables_in_a_set; + NovaConfig::config->major_compaction_type = FLAGS_major_compaction_type; + NovaConfig::config->enable_flush_multiple_memtables = FLAGS_enable_flush_multiple_memtables; + NovaConfig::config->major_compaction_max_parallism = FLAGS_major_compaction_max_parallism; + NovaConfig::config->major_compaction_max_tables_in_a_set = FLAGS_major_compaction_max_tables_in_a_set; - NovaConfig::config->number_of_recovery_threads = FLAGS_cc_num_recovery_threads; - NovaConfig::config->recover_dbs = FLAGS_cc_recover_dbs; + NovaConfig::config->number_of_recovery_threads = FLAGS_num_recovery_threads; + NovaConfig::config->recover_dbs = FLAGS_recover_dbs; - NovaConfig::config->servers = convert_hosts(FLAGS_cc_servers); - if (FLAGS_cc_local_disk) { + NovaConfig::config->servers = convert_hosts(FLAGS_all_servers); + if (FLAGS_use_local_disk) { for (int i = 0; i < NovaConfig::config->servers.size(); i++) { NovaConfig::config->ltc_servers.push_back( NovaConfig::config->servers[i]); @@ -188,7 +204,7 @@ int main(int argc, char *argv[]) { } } else { for (int i = 0; i < NovaConfig::config->servers.size(); i++) { - if (i < FLAGS_number_of_ccs) { + if (i < FLAGS_number_of_ltcs) { NovaConfig::config->ltc_servers.push_back( NovaConfig::config->servers[i]); } else { @@ -206,21 +222,21 @@ int main(int argc, char *argv[]) { for (int i = 0; i < NovaConfig::config->stoc_servers.size(); i++) { Host host = NovaConfig::config->stoc_servers[i]; NOVA_LOG(INFO) - << fmt::format("dc: {}:{}:{}", host.server_id, host.ip, host.port); + << fmt::format("stoc: {}:{}:{}", host.server_id, host.ip, host.port); } - NOVA_ASSERT(FLAGS_cc_num_log_replicas <= + NOVA_ASSERT(FLAGS_num_log_replicas <= NovaConfig::config->stoc_servers.size()); NovaConfig::config->my_server_id = FLAGS_server_id; - NovaConfig::ReadFragments(FLAGS_cc_config_path, + NovaConfig::ReadFragments(FLAGS_ltc_config_path, &NovaConfig::config->fragments); - if (FLAGS_cc_local_disk && FLAGS_server_id < FLAGS_number_of_ccs) { + if (FLAGS_use_local_disk && FLAGS_server_id < FLAGS_number_of_ltcs) { uint32_t start_stoc_id = 0; for (int i = 0; i < NovaConfig::config->fragments.size(); i++) { NovaConfig::config->fragments[i]->log_replica_stoc_ids.clear(); std::set set; - for (int r = 0; r < FLAGS_cc_num_log_replicas; r++) { + for (int r = 0; r < FLAGS_num_log_replicas; r++) { if (NovaConfig::config->stoc_servers[start_stoc_id].server_id == FLAGS_server_id) { start_stoc_id = (start_stoc_id + 1) % @@ -235,58 +251,57 @@ int main(int argc, char *argv[]) { start_stoc_id = (start_stoc_id + 1) % NovaConfig::config->stoc_servers.size(); } - NOVA_ASSERT(set.size() == FLAGS_cc_num_log_replicas); + NOVA_ASSERT(set.size() == FLAGS_num_log_replicas); NOVA_ASSERT(set.size() == NovaConfig::config->fragments[i]->log_replica_stoc_ids.size()); } } - NovaConfig::config->num_conn_workers = FLAGS_cc_num_conn_workers; - NovaConfig::config->num_fg_rdma_workers = FLAGS_cc_num_async_workers; - NovaConfig::config->num_storage_workers = FLAGS_cc_num_storage_workers; - NovaConfig::config->num_compaction_workers = FLAGS_cc_num_compaction_workers; - NovaConfig::config->num_bg_rdma_workers = FLAGS_cc_num_rdma_compaction_workers; - NovaConfig::config->num_memtables = FLAGS_cc_num_memtables; - NovaConfig::config->num_memtable_partitions = FLAGS_cc_num_memtable_partitions; - NovaConfig::config->enable_subrange = FLAGS_cc_enable_subrange; - NovaConfig::config->memtable_type = FLAGS_cc_memtable_type; - - NovaConfig::config->num_stocs_scatter_data_blocks = FLAGS_cc_rtable_num_servers_scatter_data_blocks; - NovaConfig::config->log_buf_size = FLAGS_cc_rtable_size_mb * 1024; - NovaConfig::config->max_stoc_file_size = FLAGS_cc_rtable_size_mb * 1024; - NovaConfig::config->sstable_size = FLAGS_cc_sstable_size_mb * 1024 * 1024; - NovaConfig::config->use_local_disk = FLAGS_cc_local_disk; - NovaConfig::config->num_tinyranges_per_subrange = FLAGS_cc_num_tinyranges_per_subrange; - - if (FLAGS_cc_scatter_policy == "random") { + NovaConfig::config->num_conn_workers = FLAGS_ltc_num_client_workers; + NovaConfig::config->num_fg_rdma_workers = FLAGS_num_rdma_fg_workers; + NovaConfig::config->num_storage_workers = FLAGS_num_storage_workers; + NovaConfig::config->num_compaction_workers = FLAGS_num_compaction_workers; + NovaConfig::config->num_bg_rdma_workers = FLAGS_num_rdma_bg_workers; + NovaConfig::config->num_memtables = FLAGS_num_memtables; + NovaConfig::config->num_memtable_partitions = FLAGS_num_memtable_partitions; + NovaConfig::config->enable_subrange = FLAGS_enable_subrange; + NovaConfig::config->memtable_type = FLAGS_memtable_type; + + NovaConfig::config->num_stocs_scatter_data_blocks = FLAGS_ltc_num_stocs_scatter_data_blocks; + NovaConfig::config->log_buf_size = FLAGS_max_stoc_file_size_mb * 1024; + NovaConfig::config->max_stoc_file_size = FLAGS_max_stoc_file_size_mb * 1024; + NovaConfig::config->sstable_size = FLAGS_sstable_size_mb * 1024 * 1024; + NovaConfig::config->use_local_disk = FLAGS_use_local_disk; + NovaConfig::config->num_tinyranges_per_subrange = FLAGS_num_tinyranges_per_subrange; + + if (FLAGS_scatter_policy == "random") { NovaConfig::config->scatter_policy = ScatterPolicy::RANDOM; - } else if (FLAGS_cc_scatter_policy == "power_of_two") { + } else if (FLAGS_scatter_policy == "power_of_two") { NovaConfig::config->scatter_policy = ScatterPolicy::POWER_OF_TWO; - } else if (FLAGS_cc_scatter_policy == "power_of_three") { + } else if (FLAGS_scatter_policy == "power_of_three") { NovaConfig::config->scatter_policy = ScatterPolicy::POWER_OF_THREE; } else { NovaConfig::config->scatter_policy = ScatterPolicy::SCATTER_DC_STATS; } - if (FLAGS_cc_log_record_mode == "none") { + if (FLAGS_log_record_mode == "none") { NovaConfig::config->log_record_mode = NovaLogRecordMode::LOG_NONE; - } else if (FLAGS_cc_log_record_mode == "rdma") { + } else if (FLAGS_log_record_mode == "rdma") { NovaConfig::config->log_record_mode = NovaLogRecordMode::LOG_RDMA; } - NovaConfig::config->enable_lookup_index = FLAGS_cc_enable_table_locator; - NovaConfig::config->enable_range_index = FLAGS_cc_enable_range_index; - NovaConfig::config->subrange_sampling_ratio = FLAGS_cc_sampling_ratio; - NovaConfig::config->zipfian_dist_file_path = FLAGS_cc_zipfian_dist; + NovaConfig::config->enable_lookup_index = FLAGS_enable_lookup_index; + NovaConfig::config->enable_range_index = FLAGS_enable_range_index; + NovaConfig::config->subrange_sampling_ratio = FLAGS_sampling_ratio; + NovaConfig::config->zipfian_dist_file_path = FLAGS_zipfian_dist_ref_counts; NovaConfig::config->ReadZipfianDist(); - NovaConfig::config->client_access_pattern = FLAGS_cc_client_access_pattern; - NovaConfig::config->enable_detailed_db_stats = FLAGS_cc_enable_detailed_db_stats; - NovaConfig::config->subrange_num_keys_no_flush = FLAGS_cc_subrange_no_flush_num_keys; - NovaConfig::config->l0_stop_write_mb = FLAGS_cc_l0_stop_write_mb; - NovaConfig::config->l0_start_compaction_mb = FLAGS_cc_l0_start_compaction_mb; + NovaConfig::config->client_access_pattern = FLAGS_client_access_pattern; + NovaConfig::config->enable_detailed_db_stats = FLAGS_enable_detailed_db_stats; + NovaConfig::config->subrange_num_keys_no_flush = FLAGS_subrange_no_flush_num_keys; + NovaConfig::config->l0_stop_write_mb = FLAGS_l0_stop_write_mb; + NovaConfig::config->l0_start_compaction_mb = FLAGS_l0_start_compaction_mb; NovaConfig::config->level = FLAGS_level; - NovaConfig::config->enable_subrange_reorg = FLAGS_cc_enable_subrange_reorg; - + NovaConfig::config->enable_subrange_reorg = FLAGS_enable_subrange_reorg; leveldb::EnvBGThread::bg_flush_memtable_thread_id_seq = 0; leveldb::EnvBGThread::bg_compaction_thread_id_seq = 0; nova::RDMAServerImpl::bg_storage_worker_seq_id_ = 0; diff --git a/novalsm/nova_subrange_sim_test.cpp b/novalsm/nova_subrange_sim_test.cpp index 78e9dc5..2d60907 100644 --- a/novalsm/nova_subrange_sim_test.cpp +++ b/novalsm/nova_subrange_sim_test.cpp @@ -179,9 +179,9 @@ void TestSubRanges() { NOVA_ASSERT(buf != NULL) << "Not enough memory"; system(fmt::format("exec rm -rf {}/*", NovaConfig::config->db_path).data()); system(fmt::format("exec rm -rf {}/*", - NovaConfig::config->stoc_file_path).data()); + NovaConfig::config->stoc_files_path).data()); - mkdirs(NovaConfig::config->stoc_file_path.data()); + mkdirs(NovaConfig::config->stoc_files_path.data()); mkdirs(NovaConfig::config->db_path.data()); std::vector bgs; @@ -276,7 +276,7 @@ void Read(uint32_t id) { int main(int argc, char *argv[]) { NovaConfig::config = new NovaConfig; - NovaConfig::config->stoc_file_path = "/tmp/rtables"; + NovaConfig::config->stoc_files_path = "/tmp/rtables"; NovaConfig::config->mem_pool_size_gb = 1; @@ -288,7 +288,6 @@ int main(int argc, char *argv[]) { NovaConfig::config->rdma_doorbell_batch_size = 8; NovaConfig::config->block_cache_mb = 0; - NovaConfig::config->row_cache_mb = 0; NovaConfig::config->memtable_size_mb = 16; NovaConfig::config->db_path = "/tmp/db"; diff --git a/novalsm/rdma_admission_ctrl.h b/novalsm/rdma_admission_ctrl.h index 7d226c2..26b1f35 100644 --- a/novalsm/rdma_admission_ctrl.h +++ b/novalsm/rdma_admission_ctrl.h @@ -2,7 +2,6 @@ // // Created by Haoyu Huang on 5/15/20. // Copyright (c) 2020 University of Southern California. All rights reserved. -// #ifndef LEVELDB_RDMA_ADMISSION_CTRL_H #define LEVELDB_RDMA_ADMISSION_CTRL_H @@ -10,6 +9,7 @@ #include "common/nova_config.h" namespace nova { + // We maintain RDMA buffer as a circular buffer. This ensures we don't send too many requests that overflow the buffer. class RDMAAdmissionCtrl { public: RDMAAdmissionCtrl() : max_pending_rdma_requests_per_endpoint_( @@ -22,7 +22,7 @@ namespace nova { bool CanIssueRequest(int server_id); - bool CanIssueRequest(const std::vector& server_ids); + bool CanIssueRequest(const std::vector &server_ids); void RemoveRequests(int server_id, int requests); diff --git a/novalsm/rdma_msg_handler.h b/novalsm/rdma_msg_handler.h index 314efcc..7f44497 100644 --- a/novalsm/rdma_msg_handler.h +++ b/novalsm/rdma_msg_handler.h @@ -2,17 +2,17 @@ // // Created by Haoyu Huang on 12/25/19. // Copyright (c) 2019 University of Southern California. All rights reserved. -// +// RDMA request handler. -#ifndef LEVELDB_RDMA_MSG_HANDLER_H -#define LEVELDB_RDMA_MSG_HANDLER_H +#ifndef RDMA_MSG_HANDLER_H +#define RDMA_MSG_HANDLER_H #include #include #include #include "leveldb/options.h" #include "common/nova_common.h" -#include "rdma/nova_msg_callback.h" +#include "rdma/rdma_msg_callback.h" #include "rdma/nova_rdma_broker.h" #include #include @@ -31,7 +31,7 @@ namespace nova { class RDMAServerImpl; - class RDMAMsgHandler : public NovaMsgCallback { + class RDMAMsgHandler : public RDMAMsgCallback { public: RDMAMsgHandler(RdmaCtrl *rdma_ctrl, NovaMemManager *mem_manager, @@ -99,4 +99,4 @@ namespace nova { } -#endif //LEVELDB_RDMA_MSG_HANDLER_H +#endif //RDMA_MSG_HANDLER_H diff --git a/novalsm/rdma_server.cpp b/novalsm/rdma_server.cpp index d80f89b..c4f29df 100644 --- a/novalsm/rdma_server.cpp +++ b/novalsm/rdma_server.cpp @@ -240,7 +240,7 @@ namespace nova { mem_manager_->FreeItem(thread_id_, allocated_buf, scid); processed = true; NOVA_LOG(DEBUG) << fmt::format( - "dc[{}]: imm:{} type:{} allocated buf:{} size:{} wr:{}.", + "rdma-server[{}]: imm:{} type:{} allocated buf:{} size:{} wr:{}.", thread_id_, imm_data, buf[0], allocated_buf_int, size, wr_id); @@ -267,7 +267,7 @@ namespace nova { context.stoc_file_id, context.stoc_file_buf_offset)) << fmt::format( - "dc[{}]: Write StoC file failed id:{} offset:{} creq_id:{} req_id:{}", + "rdma-server[{}]: Write StoC file failed id:{} offset:{} creq_id:{} req_id:{}", thread_id_, context.stoc_file_id, context.stoc_file_buf_offset, stoc_req_id, @@ -275,7 +275,7 @@ namespace nova { processed = true; NOVA_LOG(DEBUG) << fmt::format( - "dc[{}]: Write StoC file complete id:{} offset:{} creq_id:{} req_id:{}", + "rdma-server[{}]: Write StoC file complete id:{} offset:{} creq_id:{} req_id:{}", thread_id_, context.stoc_file_id, context.stoc_file_buf_offset, stoc_req_id, @@ -333,7 +333,7 @@ namespace nova { } } NOVA_LOG(DEBUG) << fmt::format( - "dc[{}]: Delete SSTables. nsstables:{}", + "rdma-server[{}]: Delete SSTables. nsstables:{}", thread_id_, nfiles); processed = true; } else if (buf[0] == @@ -357,7 +357,7 @@ namespace nova { std::string filename; leveldb::DecodeStr(buf + msg_size, &filename); NOVA_LOG(DEBUG) << fmt::format( - "dc{}: Read blocks of StoC file {} offset:{} size:{} ltc_mr_offset:{} file:{}", + "rdma-server{}: Read blocks of StoC file {} offset:{} size:{} ltc_mr_offset:{} file:{}", thread_id_, stoc_file_id, offset, size, ltc_mr_offset, filename); @@ -428,10 +428,10 @@ namespace nova { uint64_t stoc_file_off = stoc_file->AllocateBuf( filename, size, is_meta_blocks); NOVA_ASSERT(stoc_file_off != UINT64_MAX) - << fmt::format("dc{}: {} {}", thread_id_, filename, + << fmt::format("rdma-server{}: {} {}", thread_id_, filename, size); NOVA_ASSERT(stoc_file->stoc_file_name_ == filename) - << fmt::format("dc{}: {} {}", thread_id_, + << fmt::format("rdma-server{}: {} {}", thread_id_, stoc_file->stoc_file_name_, filename); @@ -453,7 +453,7 @@ namespace nova { request_context_map_[req_id] = context; NOVA_LOG(DEBUG) << fmt::format( - "dc{}: Allocate buf for StoC file Write db:{} fn:{} size:{} file_id:{} file_off:{} fname:{}", + "rdma-server{}: Allocate buf for StoC file Write db:{} fn:{} size:{} file_id:{} file_off:{} fname:{}", thread_id_, dbname, file_number, size, stoc_file->file_id(), stoc_file_off, filename); processed = true; @@ -476,7 +476,7 @@ namespace nova { private_cq_.push_back(task); NOVA_LOG(DEBUG) << fmt::format( - "dc[{}]: Allocate log buffer for file {}.", + "rdma-server{}]: Allocate log buffer for file {}.", thread_id_, log_file); processed = true; } else if (buf[0] == @@ -502,7 +502,7 @@ namespace nova { context.size = size; request_context_map_[req_id] = context; NOVA_LOG(DEBUG) << fmt::format( - "dc[{}]: Allocate buffer for RDMA WRITE.", + "rdma-server{}]: Allocate buffer for RDMA WRITE.", thread_id_); processed = true; } else if (buf[0] == @@ -540,7 +540,7 @@ namespace nova { } log_manager_->DeleteLogBuf(logfiles); NOVA_LOG(DEBUG) << fmt::format( - "dc[{}]: Delete log buffer for file {}.", + "rdma-server{}]: Delete log buffer for file {}.", thread_id_, logfiles.size()); processed = true; } else if (buf[0] == @@ -565,7 +565,7 @@ namespace nova { task.stoc_req_id = stoc_req_id; private_cq_.push_back(task); NOVA_LOG(DEBUG) << fmt::format( - "dc[{}]: Filename stoc file mapping {}.", + "rdma-server{}]: Filename stoc file mapping {}.", thread_id_, fn_stocfile.size()); processed = true; } else if (buf[0] == diff --git a/novalsm/rdma_server.h b/novalsm/rdma_server.h index 02d7278..326838e 100644 --- a/novalsm/rdma_server.h +++ b/novalsm/rdma_server.h @@ -2,10 +2,9 @@ // // Created by Haoyu Huang on 1/8/20. // Copyright (c) 2020 University of Southern California. All rights reserved. -// -#ifndef LEVELDB_RDMA_SERVER_H -#define LEVELDB_RDMA_SERVER_H +#ifndef RDMA_SERVER_H +#define RDMA_SERVER_H #include #include @@ -79,7 +78,8 @@ namespace nova { bool is_meta_blocks; }; - class RDMAServerImpl : public NovaMsgCallback, public leveldb::RDMAServer { + // RDMA server class that handles RDMA client requests. + class RDMAServerImpl : public RDMAMsgCallback, public leveldb::RDMAServer { public: RDMAServerImpl(rdmaio::RdmaCtrl *rdma_ctrl, NovaMemManager *mem_manager, @@ -135,4 +135,4 @@ namespace nova { } -#endif //LEVELDB_RDMA_SERVER_H +#endif //RDMA_SERVER_H diff --git a/rdma/nova_msg_callback.h b/rdma/nova_msg_callback.h deleted file mode 100644 index 09351f3..0000000 --- a/rdma/nova_msg_callback.h +++ /dev/null @@ -1,21 +0,0 @@ - -// -// Created by Haoyu Huang on 4/1/19. -// Copyright (c) 2019 University of Southern California. All rights reserved. -// - -#ifndef RLIB_NOVA_MSG_CALLBACK_H -#define RLIB_NOVA_MSG_CALLBACK_H - -#include - -namespace nova { - class NovaMsgCallback { - public: - virtual bool - ProcessRDMAWC(ibv_wc_opcode type, uint64_t wr_id, int remote_server_id, - char *buf, uint32_t imm_data, bool* generate_a_new_request) = 0; - - }; -} -#endif //RLIB_NOVA_MSG_CALLBACK_H diff --git a/rdma/nova_rdma_broker.h b/rdma/nova_rdma_broker.h index b7dcb20..c6c5379 100644 --- a/rdma/nova_rdma_broker.h +++ b/rdma/nova_rdma_broker.h @@ -46,7 +46,7 @@ namespace nova { virtual char *GetSendBuf(int server_id) = 0; - virtual uint32_t store_id() = 0; + virtual uint32_t broker_id() = 0; virtual const std::vector &end_points() = 0; }; @@ -82,7 +82,7 @@ namespace nova { char *GetSendBuf(int server_id) { return NULL; } - uint32_t store_id() { return 0; } + uint32_t broker_id() { return 0; } const std::vector &end_points() { return dummy_; } std::vector dummy_; diff --git a/rdma/nova_rdma_rc_broker.h b/rdma/nova_rdma_rc_broker.h index 588ae9c..56bce94 100644 --- a/rdma/nova_rdma_rc_broker.h +++ b/rdma/nova_rdma_rc_broker.h @@ -11,7 +11,7 @@ #include "rdma_ctrl.hpp" #include "nova_rdma_broker.h" -#include "nova_msg_callback.h" +#include "rdma_msg_callback.h" #include "common/nova_common.h" @@ -19,7 +19,8 @@ namespace nova { using namespace rdmaio; - // Thread local. One thread has one RDMA RC Store. + // Thread local. One thread has one RDMA RC Broker. + // It maintains a circular buffer to issue RDMA SENDs. class NovaRDMARCBroker : public NovaRDMABroker { public: NovaRDMARCBroker(char *buf, int thread_id, @@ -32,7 +33,7 @@ namespace nova { char *mr_buf, uint64_t mr_size, uint64_t rdma_port, - NovaMsgCallback *callback) : + RDMAMsgCallback *callback) : rdma_buf_(buf), thread_id_(thread_id), end_points_(end_points), @@ -133,7 +134,7 @@ namespace nova { char *GetSendBuf(int server_id); - uint32_t store_id() { return thread_id_; } + uint32_t broker_id() { return thread_id_; } void ReinitializeQPs(rdmaio::RdmaCtrl *rdma_ctrl); @@ -179,7 +180,7 @@ namespace nova { int *send_sge_index_ = nullptr; int *npending_send_ = nullptr; int *psend_index_ = nullptr; - NovaMsgCallback *callback_ = nullptr; + RDMAMsgCallback *callback_ = nullptr; void InitializeQPs(RdmaCtrl *rdma_ctrl); diff --git a/rdma/rdma_msg_callback.h b/rdma/rdma_msg_callback.h new file mode 100644 index 0000000..56370b1 --- /dev/null +++ b/rdma/rdma_msg_callback.h @@ -0,0 +1,23 @@ + +// +// Created by Haoyu Huang on 4/1/19. +// Copyright (c) 2019 University of Southern California. All rights reserved. +// + +#ifndef RDMA_MSG_CALLBACK_H +#define RDMA_MSG_CALLBACK_H + +#include + +namespace nova { + // After calling ibv_poll_cq, this function is invoked for each completion event. + class RDMAMsgCallback { + public: + virtual bool + ProcessRDMAWC(ibv_wc_opcode type, uint64_t wr_id, int remote_server_id, + char *buf, uint32_t imm_data, + bool *generate_a_new_request) = 0; + + }; +} +#endif //RDMA_MSG_CALLBACK_H diff --git a/stoc/persistent_stoc_file.h b/stoc/persistent_stoc_file.h index afd90f1..ee5f4cf 100644 --- a/stoc/persistent_stoc_file.h +++ b/stoc/persistent_stoc_file.h @@ -16,6 +16,7 @@ namespace leveldb { + // Persistent StoC file. class StoCPersistentFile { public: StoCPersistentFile(uint32_t file_id, Env *env, std::string filename, diff --git a/stoc/storage_worker.h b/stoc/storage_worker.h index dcc3271..544465b 100644 --- a/stoc/storage_worker.h +++ b/stoc/storage_worker.h @@ -23,6 +23,7 @@ namespace nova { class StorageTask; class ServerCompleteTask; + // A storage worker that handles storage related requests. class StorageWorker : public leveldb::EnvBGThread { public: StorageWorker(leveldb::StocPersistentFileManager *stoc_file_manager,