Optimize snapshot creation by excluding pre-sorted files

project-tsurugi · Oct 2, 2024 · 8209c50 · 8209c50
1 parent aed365d
commit 8209c50
Show file tree

Hide file tree

Showing 5 changed files with 18 additions and 17 deletions.
diff --git a/include/limestone/api/cursor.h b/include/limestone/api/cursor.h
@@ -58,7 +58,7 @@ class cursor {
      * @brief returns the storage ID of the entry at the current cursor position
      * @return the storage ID of the current entry
      */
-    storage_id_type storage() const noexcept;
+    [[nodiscard]] storage_id_type storage() const noexcept;
 
     /**
      * @brief returns the key byte string of the entry at the current cursor position

diff --git a/include/limestone/api/snapshot.h b/include/limestone/api/snapshot.h
@@ -73,11 +73,9 @@ class snapshot {
     [[nodiscard]] std::unique_ptr<cursor> scan(storage_id_type storage_id, std::string_view entry_key, bool inclusive) const noexcept;
 
 private:
-    boost::filesystem::path dir_{};
+    boost::filesystem::path location_{};
 
-    [[nodiscard]] boost::filesystem::path file_path() const noexcept;
-
-    explicit snapshot(const boost::filesystem::path& location) noexcept;
+    explicit snapshot(const boost::filesystem::path location) noexcept;
 
     friend class datastore;
 };

diff --git a/src/limestone/cursor.cpp b/src/limestone/cursor.cpp
@@ -170,9 +170,9 @@ cursor::~cursor() noexcept {
 }
 
 bool cursor::next() {
-    // Keep calling next() until we find a non-remove_entry, or no more entries
+    // Keep calling next() until we find a noraml_entry, or no more entries
     while (log_entry_tracker_->next()) {
-        if (log_entry_tracker_->type() != log_entry::entry_type::remove_entry) {
+        if (log_entry_tracker_->type() == limestone::api::log_entry::entry_type::normal_entry) {
             return true;  
         }
     }

diff --git a/src/limestone/datastore_snapshot.cpp b/src/limestone/datastore_snapshot.cpp
@@ -325,8 +325,7 @@ std::set<std::string> assemble_snapshot_input_filenames(
         }
         if (boost::filesystem::is_regular_file(it->path())) {
             std::string filename = it->path().filename().string();
-//            if (detached_pwals.find(filename) == detached_pwals.end() && filename != compaction_catalog::get_compacted_filename()) {
-            if (detached_pwals.find(filename) == detached_pwals.end()) {
+            if (detached_pwals.find(filename) == detached_pwals.end() && filename != compaction_catalog::get_compacted_filename()) {
                 filename_set.insert(filename);
             }
         }
@@ -365,7 +364,9 @@ void datastore::create_snapshot() {
     }
     setvbuf(ostrm, nullptr, _IOFBF, 128L * 1024L);  // NOLINT, NB. glibc may ignore size when _IOFBF and buffer=NULL
     auto write_snapshot_entry = [&ostrm](std::string_view key, std::string_view value){log_entry::write(ostrm, key, value);};
-    sortdb_foreach(sctx, write_snapshot_entry, true);
+
+    bool skip_remove_entry = compaction_catalog_->get_compacted_files().empty();
+    sortdb_foreach(sctx, write_snapshot_entry, skip_remove_entry);
     if (fclose(ostrm) != 0) {  // NOLINT(*-owning-memory)
         LOG_AND_THROW_IO_EXCEPTION("cannot close snapshot file (" + snapshot_file.string() + ")", errno);
     }

diff --git a/src/limestone/snapshot.cpp b/src/limestone/snapshot.cpp
@@ -17,15 +17,21 @@
 
 #include <glog/logging.h>
 #include <limestone/logging.h>
+#include "compaction_catalog.h"
 #include "logging_helper.h"
 
 namespace limestone::api {  // FIXME fill implementation
 
-snapshot::snapshot(const boost::filesystem::path& location) noexcept : dir_(location / boost::filesystem::path(std::string(subdirectory_name_))) {
-}
+snapshot::snapshot(boost::filesystem::path location) noexcept : location_(std::move(location)) {}
 
 std::unique_ptr<cursor> snapshot::get_cursor() const {
-    return std::unique_ptr<cursor>(new cursor(file_path()));
+    boost::filesystem::path compacted_file = location_ / limestone::internal::compaction_catalog::get_compacted_filename();
+    boost::filesystem::path snapshot_file = location_ / std::string(subdirectory_name_) / std::string(file_name_);
+
+    if (boost::filesystem::exists(compacted_file)) {
+        return std::unique_ptr<cursor>(new cursor(snapshot_file, compacted_file));
+    }
+    return std::unique_ptr<cursor>(new cursor(snapshot_file));
 }
 
 std::unique_ptr<cursor> snapshot::find([[maybe_unused]] storage_id_type storage_id, [[maybe_unused]] std::string_view entry_key) const noexcept {
@@ -38,8 +44,4 @@ std::unique_ptr<cursor> snapshot::scan([[maybe_unused]] storage_id_type storage_
     std::abort();  // FIXME should implement
 }
 
-boost::filesystem::path snapshot::file_path() const noexcept {
-    return dir_ / boost::filesystem::path(std::string(file_name_));
-}
-
 } // namespace limestone::api