Wip/log 0.7 (#57)

feat(performance): optimize startup time by eliminating unnecessary sorting

This commit improves startup performance by removing unnecessary sorting operations during system startup. Redundant file handling has been excluded, and the snapshot and compacted file processing logic has been optimized.

include/limestone/api/cursor.h

@@ -17,18 +17,25 @@
 
 #include <memory>
 #include <vector>
+#include <optional> 
 
 #include <boost/filesystem.hpp>
 #include <boost/filesystem/fstream.hpp>
 
 #include <limestone/api/storage_id_type.h>
 #include <limestone/api/large_object_view.h>
 
+
+namespace limestone::internal {
+    class cursor_impl;  
+}
+
 namespace limestone::api {
 
 class log_entry;
 class snapshot;
 
+
 /**
  * @brief a cursor to scan entries on the snapshot
  */
@@ -56,7 +63,7 @@ class cursor {
      * @brief returns the storage ID of the entry at the current cursor position
      * @return the storage ID of the current entry
      */
-    storage_id_type storage() const noexcept;
+    [[nodiscard]] storage_id_type storage() const noexcept;
 
     /**
      * @brief returns the key byte string of the entry at the current cursor position
@@ -77,13 +84,14 @@ class cursor {
     std::vector<large_object_view>& large_objects() noexcept;
 
 private:
-    boost::filesystem::ifstream istrm_{};
-    std::unique_ptr<log_entry> log_entry_;
+    std::unique_ptr<internal::cursor_impl> pimpl;
+
     std::vector<large_object_view> large_objects_{};
 
-    explicit cursor(const boost::filesystem::path& file);
-
-    friend class snapshot;
+    explicit cursor(const boost::filesystem::path& snapshot_file);
+    explicit cursor(const boost::filesystem::path& snapshot_file, const boost::filesystem::path& compacted_file);
+
+    friend class internal::cursor_impl;
 };
 
 } // namespace limestone::api

include/limestone/api/datastore.h

@@ -23,6 +23,7 @@
 #include <set>
 #include <mutex>
 #include <queue>
+#include <map>
 
 #include <boost/filesystem.hpp>
 
@@ -309,7 +310,7 @@ class datastore {
      * @param from the location of log files
      * @attention this function is not thread-safe.
      */
-    void create_snapshot(const std::set<std::string>& file_names);
+    void create_snapshot();
 
     epoch_id_type last_durable_epoch_in_dir();
 
@@ -324,7 +325,8 @@ class datastore {
     void rotate_epoch_file();
 
     int64_t current_unix_epoch_in_millis();
-
+
+    std::map<storage_id_type, write_version_type> clear_storage;  
 };
 
 } // namespace limestone::api

include/limestone/api/log_channel.h

@@ -41,7 +41,6 @@ class rotation_result;
  * @details this object is not thread-safe, assuming each thread uses its own log_channel
  */
 class log_channel {
-    friend class rotation_task;
 
 public:
     /**
@@ -180,6 +179,7 @@ class log_channel {
     rotation_result do_rotate_file(epoch_id_type epoch = 0);
 
     friend class datastore;
+    friend class rotation_task;
 };
 
 } // namespace limestone::api

include/limestone/api/snapshot.h

@@ -17,19 +17,30 @@
 
 #include <memory>
 #include <string_view>
-
+#include <map>
 #include <boost/filesystem.hpp>
 
 #include <limestone/api/cursor.h>
+#include <limestone/api/write_version_type.h>
+
+namespace limestone::internal {
+    class snapshot_impl;  
+}
 
 namespace limestone::api {
 
 /**
  * @brief a snapshot of the data at a point in time on the data store
  */
 class snapshot {
+
 public:
     snapshot() noexcept = delete;
+    snapshot(const snapshot&) = delete;
+    snapshot& operator=(const snapshot&) = delete;
+    snapshot(snapshot&&) noexcept = delete;
+    snapshot& operator=(snapshot&&) noexcept = delete;
+    ~snapshot();
 
     /**
      * @brief directory name of a snapshot
@@ -73,11 +84,9 @@ class snapshot {
     [[nodiscard]] std::unique_ptr<cursor> scan(storage_id_type storage_id, std::string_view entry_key, bool inclusive) const noexcept;
 
 private:
-    boost::filesystem::path dir_{};
-
-    [[nodiscard]] boost::filesystem::path file_path() const noexcept;
+    std::unique_ptr<internal::snapshot_impl> pimpl;
 
-    explicit snapshot(const boost::filesystem::path& location) noexcept;
+    explicit snapshot(boost::filesystem::path location, std::map<storage_id_type, write_version_type> clear_storage) noexcept;
 
     friend class datastore;
 };

src/limestone/cursor.cpp

@@ -20,43 +20,37 @@
 #include "logging_helper.h"
 #include "limestone_exception_helper.h"
 #include "log_entry.h"
+#include "cursor_impl.h"
+
 
 namespace limestone::api {
 
-cursor::cursor(const boost::filesystem::path& file) : log_entry_(std::make_unique<log_entry>()) {
-    istrm_.open(file, std::ios_base::in | std::ios_base::binary );
-    if (!istrm_.good()) {
-        LOG_AND_THROW_EXCEPTION("file stream of the cursor is not good (" + file.string() + ")");
-    }
-}
+
+cursor::cursor(const boost::filesystem::path& snapshot_file)
+    : pimpl(std::make_unique<limestone::internal::cursor_impl>(snapshot_file)) {}
+
+cursor::cursor(const boost::filesystem::path& snapshot_file, const boost::filesystem::path& compacted_file)
+    : pimpl(std::make_unique<limestone::internal::cursor_impl>(snapshot_file, compacted_file)) {}
+
 cursor::~cursor() noexcept {
-    istrm_.close();
+    // TODO: handle close failure
+    pimpl->close();
 }
 
 bool cursor::next() {
-    if (!istrm_.good()) {
-        DVLOG_LP(log_trace) << "file stream of the cursor is not good";
-        return false;
-    }
-    if (istrm_.eof()) {
-        DVLOG_LP(log_trace) << "already detected eof of the cursor";
-        return false;
-    }
-    auto rv = log_entry_->read(istrm_);
-    DVLOG_LP(log_trace) << (rv ? "read an entry from the cursor" : "detect eof of the cursor");
-    return rv;
+    return pimpl->next();
 }
 
 storage_id_type cursor::storage() const noexcept {
-    return log_entry_->storage();
+    return pimpl->storage();
 }
 
 void cursor::key(std::string& buf) const noexcept {
-    log_entry_->key(buf);
+    pimpl->key(buf);
 }
 
 void cursor::value(std::string& buf) const noexcept {
-    log_entry_->value(buf);
+    pimpl->value(buf);
 }
 
 std::vector<large_object_view>& cursor::large_objects() noexcept {