diff --git a/build.sh b/build.sh index 1da3d89..92514bf 100644 --- a/build.sh +++ b/build.sh @@ -33,7 +33,7 @@ cd $BUILD_PATH && wget https://github.com/facebook/zstd/archive/v${zstd_version} -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DZSTD_ZLIB_SUPPORT=ON -DZSTD_LZMA_SUPPORT=OFF -DCMAKE_BUILD_TYPE=Release .. && make -j$(nproc) install && \ cd $BUILD_PATH && rm -rf * && ldconfig -rocksdb_version="6.16.4" +rocksdb_version="6.17.3" cd $BUILD_PATH && wget https://github.com/facebook/rocksdb/archive/v${rocksdb_version}.tar.gz && tar xzf v${rocksdb_version}.tar.gz && cd rocksdb-${rocksdb_version}/ && \ mkdir -p build_place && cd build_place && cmake -DCMAKE_BUILD_TYPE=Release $CMAKE_REQUIRED_PARAMS -DCMAKE_PREFIX_PATH=$INSTALL_PREFIX -DWITH_TESTS=OFF -DWITH_GFLAGS=OFF \ -DWITH_BENCHMARK_TOOLS=OFF -DWITH_TOOLS=OFF -DWITH_MD_LIBRARY=OFF -DWITH_RUNTIME_DEBUG=OFF -DROCKSDB_BUILD_SHARED=OFF -DWITH_SNAPPY=ON -DWITH_LZ4=ON -DWITH_ZLIB=ON \ diff --git a/db.go b/db.go index 81c9643..ad2ca7c 100755 --- a/db.go +++ b/db.go @@ -931,10 +931,10 @@ func (db *DB) DropColumnFamily(c *ColumnFamilyHandle) (err error) { // // The keys counted will begin at Range.Start and end on the key before // Range.Limit. -func (db *DB) GetApproximateSizes(ranges []Range) []uint64 { +func (db *DB) GetApproximateSizes(ranges []Range) ([]uint64, error) { sizes := make([]uint64, len(ranges)) if len(ranges) == 0 { - return sizes + return sizes, nil } cStarts := make([]*C.char, len(ranges)) @@ -948,6 +948,7 @@ func (db *DB) GetApproximateSizes(ranges []Range) []uint64 { cLimitLens[i] = C.size_t(len(r.Limit)) } + var cErr *C.char C.rocksdb_approximate_sizes( db.c, C.int(len(ranges)), @@ -955,7 +956,12 @@ func (db *DB) GetApproximateSizes(ranges []Range) []uint64 { &cStartLens[0], &cLimits[0], &cLimitLens[0], - (*C.uint64_t)(&sizes[0])) + (*C.uint64_t)(&sizes[0]), + &cErr, + ) + + // parse error + err := fromCError(cErr) // free before return for i := range ranges { @@ -963,7 +969,7 @@ func (db *DB) GetApproximateSizes(ranges []Range) []uint64 { C.free(unsafe.Pointer(cLimits[i])) } - return sizes + return sizes, err } // GetApproximateSizesCF returns the approximate number of bytes of file system @@ -971,10 +977,10 @@ func (db *DB) GetApproximateSizes(ranges []Range) []uint64 { // // The keys counted will begin at Range.Start and end on the key before // Range.Limit. -func (db *DB) GetApproximateSizesCF(cf *ColumnFamilyHandle, ranges []Range) []uint64 { +func (db *DB) GetApproximateSizesCF(cf *ColumnFamilyHandle, ranges []Range) ([]uint64, error) { sizes := make([]uint64, len(ranges)) if len(ranges) == 0 { - return sizes + return sizes, nil } cStarts := make([]*C.char, len(ranges)) @@ -988,6 +994,7 @@ func (db *DB) GetApproximateSizesCF(cf *ColumnFamilyHandle, ranges []Range) []ui cLimitLens[i] = C.size_t(len(r.Limit)) } + var cErr *C.char C.rocksdb_approximate_sizes_cf( db.c, cf.c, @@ -996,7 +1003,12 @@ func (db *DB) GetApproximateSizesCF(cf *ColumnFamilyHandle, ranges []Range) []ui &cStartLens[0], &cLimits[0], &cLimitLens[0], - (*C.uint64_t)(&sizes[0])) + (*C.uint64_t)(&sizes[0]), + &cErr, + ) + + // parse error + err := fromCError(cErr) // free before return for i := range ranges { @@ -1004,7 +1016,7 @@ func (db *DB) GetApproximateSizesCF(cf *ColumnFamilyHandle, ranges []Range) []ui C.free(unsafe.Pointer(cLimits[i])) } - return sizes + return sizes, err } // SetOptions dynamically changes options through the SetOptions API. diff --git a/db_test.go b/db_test.go index fe0137b..6929014 100755 --- a/db_test.go +++ b/db_test.go @@ -263,16 +263,19 @@ func TestDBGetApproximateSizes(t *testing.T) { defer db.Close() // no ranges - sizes := db.GetApproximateSizes(nil) + sizes, err := db.GetApproximateSizes(nil) require.EqualValues(t, len(sizes), 0) + require.NoError(t, err) // range will nil start and limit - sizes = db.GetApproximateSizes([]Range{{Start: nil, Limit: nil}}) + sizes, err = db.GetApproximateSizes([]Range{{Start: nil, Limit: nil}}) require.EqualValues(t, sizes, []uint64{0}) + require.NoError(t, err) // valid range - sizes = db.GetApproximateSizes([]Range{{Start: []byte{0x00}, Limit: []byte{0xFF}}}) + sizes, err = db.GetApproximateSizes([]Range{{Start: []byte{0x00}, Limit: []byte{0xFF}}}) require.EqualValues(t, sizes, []uint64{0}) + require.NoError(t, err) } func TestDBGetApproximateSizesCF(t *testing.T) { @@ -285,14 +288,17 @@ func TestDBGetApproximateSizesCF(t *testing.T) { require.Nil(t, err) // no ranges - sizes := db.GetApproximateSizesCF(cf, nil) + sizes, err := db.GetApproximateSizesCF(cf, nil) require.EqualValues(t, len(sizes), 0) + require.NoError(t, err) // range will nil start and limit - sizes = db.GetApproximateSizesCF(cf, []Range{{Start: nil, Limit: nil}}) + sizes, err = db.GetApproximateSizesCF(cf, []Range{{Start: nil, Limit: nil}}) require.EqualValues(t, sizes, []uint64{0}) + require.NoError(t, err) // valid range - sizes = db.GetApproximateSizesCF(cf, []Range{{Start: []byte{0x00}, Limit: []byte{0xFF}}}) + sizes, err = db.GetApproximateSizesCF(cf, []Range{{Start: []byte{0x00}, Limit: []byte{0xFF}}}) require.EqualValues(t, sizes, []uint64{0}) + require.NoError(t, err) } diff --git a/dist/linux_amd64/include/rocksdb/c.h b/dist/linux_amd64/include/rocksdb/c.h index 8c3668e..ed3382a 100644 --- a/dist/linux_amd64/include/rocksdb/c.h +++ b/dist/linux_amd64/include/rocksdb/c.h @@ -496,13 +496,13 @@ extern ROCKSDB_LIBRARY_API char* rocksdb_property_value_cf( extern ROCKSDB_LIBRARY_API void rocksdb_approximate_sizes( rocksdb_t* db, int num_ranges, const char* const* range_start_key, const size_t* range_start_key_len, const char* const* range_limit_key, - const size_t* range_limit_key_len, uint64_t* sizes); + const size_t* range_limit_key_len, uint64_t* sizes, char** errptr); extern ROCKSDB_LIBRARY_API void rocksdb_approximate_sizes_cf( rocksdb_t* db, rocksdb_column_family_handle_t* column_family, int num_ranges, const char* const* range_start_key, const size_t* range_start_key_len, const char* const* range_limit_key, - const size_t* range_limit_key_len, uint64_t* sizes); + const size_t* range_limit_key_len, uint64_t* sizes, char** errptr); extern ROCKSDB_LIBRARY_API void rocksdb_compact_range(rocksdb_t* db, const char* start_key, @@ -1981,7 +1981,7 @@ extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get_for_update( const char* key, size_t klen, size_t* vlen, unsigned char exclusive, char** errptr); -char* rocksdb_transaction_get_for_update_cf( +extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get_for_update_cf( rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, rocksdb_column_family_handle_t* column_family, const char* key, size_t klen, size_t* vlen, unsigned char exclusive, char** errptr); diff --git a/dist/linux_amd64/include/rocksdb/compaction_filter.h b/dist/linux_amd64/include/rocksdb/compaction_filter.h index ed17889..9ffd776 100644 --- a/dist/linux_amd64/include/rocksdb/compaction_filter.h +++ b/dist/linux_amd64/include/rocksdb/compaction_filter.h @@ -83,7 +83,7 @@ class CompactionFilter { // // Note that RocksDB snapshots (i.e. call GetSnapshot() API on a // DB* object) will not guarantee to preserve the state of the DB with - // CompactionFilter. Data seen from a snapshot might disppear after a + // CompactionFilter. Data seen from a snapshot might disappear after a // compaction finishes. If you use snapshots, think twice about whether you // want to use compaction filter and whether you are using it in a safe way. // @@ -188,7 +188,7 @@ class CompactionFilter { // This function is deprecated. Snapshots will always be ignored for // compaction filters, because we realized that not ignoring snapshots doesn't - // provide the gurantee we initially thought it would provide. Repeatable + // provide the guarantee we initially thought it would provide. Repeatable // reads will not be guaranteed anyway. If you override the function and // returns false, we will fail the compaction. virtual bool IgnoreSnapshots() const { return true; } diff --git a/dist/linux_amd64/include/rocksdb/configurable.h b/dist/linux_amd64/include/rocksdb/configurable.h index 95c5cf4..f6f5cc7 100644 --- a/dist/linux_amd64/include/rocksdb/configurable.h +++ b/dist/linux_amd64/include/rocksdb/configurable.h @@ -240,7 +240,7 @@ class Configurable { // @param config_options Controls how the object is prepared. Also contains // a Logger and Env that can be used to initialize this object. // @return OK If the object was successfully initialized. - // @return InvalidArgument If this object could not be successfull + // @return InvalidArgument If this object could not be successfully // initialized. virtual Status PrepareOptions(const ConfigOptions& config_options); diff --git a/dist/linux_amd64/include/rocksdb/db.h b/dist/linux_amd64/include/rocksdb/db.h index 995d9f0..ee5f4ac 100644 --- a/dist/linux_amd64/include/rocksdb/db.h +++ b/dist/linux_amd64/include/rocksdb/db.h @@ -1027,20 +1027,22 @@ class DB { // Simpler versions of the GetApproximateSizes() method above. // The include_flags argumenbt must of type DB::SizeApproximationFlags // and can not be NONE. - virtual void GetApproximateSizes(ColumnFamilyHandle* column_family, - const Range* ranges, int n, uint64_t* sizes, - uint8_t include_flags = INCLUDE_FILES) { + virtual Status GetApproximateSizes(ColumnFamilyHandle* column_family, + const Range* ranges, int n, + uint64_t* sizes, + uint8_t include_flags = INCLUDE_FILES) { SizeApproximationOptions options; options.include_memtabtles = (include_flags & SizeApproximationFlags::INCLUDE_MEMTABLES) != 0; options.include_files = (include_flags & SizeApproximationFlags::INCLUDE_FILES) != 0; - Status s = GetApproximateSizes(options, column_family, ranges, n, sizes); - s.PermitUncheckedError(); + return GetApproximateSizes(options, column_family, ranges, n, sizes); } - virtual void GetApproximateSizes(const Range* ranges, int n, uint64_t* sizes, - uint8_t include_flags = INCLUDE_FILES) { - GetApproximateSizes(DefaultColumnFamily(), ranges, n, sizes, include_flags); + virtual Status GetApproximateSizes(const Range* ranges, int n, + uint64_t* sizes, + uint8_t include_flags = INCLUDE_FILES) { + return GetApproximateSizes(DefaultColumnFamily(), ranges, n, sizes, + include_flags); } // The method is similar to GetApproximateSizes, except it diff --git a/dist/linux_amd64/include/rocksdb/env.h b/dist/linux_amd64/include/rocksdb/env.h index a129b19..f8eec96 100644 --- a/dist/linux_amd64/include/rocksdb/env.h +++ b/dist/linux_amd64/include/rocksdb/env.h @@ -30,6 +30,7 @@ // Windows API macro interference #undef DeleteFile #undef GetCurrentTime +#undef LoadLibrary #endif #if defined(__GNUC__) || defined(__clang__) @@ -282,7 +283,8 @@ class Env { virtual Status FileExists(const std::string& fname) = 0; // Store in *result the names of the children of the specified directory. - // The names are relative to "dir". + // The names are relative to "dir", and shall never include the + // names `.` or `..`. // Original contents of *results are dropped. // Returns OK if "dir" exists and "*result" contains its children. // NotFound if "dir" does not exist, the calling process does not have @@ -295,7 +297,8 @@ class Env { // In case the implementation lists the directory prior to iterating the files // and files are concurrently deleted, the deleted files will be omitted from // result. - // The name attributes are relative to "dir". + // The name attributes are relative to "dir", and shall never include the + // names `.` or `..`. // Original contents of *results are dropped. // Returns OK if "dir" exists and "*result" contains its children. // NotFound if "dir" does not exist, the calling process does not have @@ -1663,6 +1666,6 @@ Env* NewTimedEnv(Env* base_env); Status NewEnvLogger(const std::string& fname, Env* env, std::shared_ptr* result); -std::unique_ptr NewCompositeEnv(std::shared_ptr fs); +std::unique_ptr NewCompositeEnv(const std::shared_ptr& fs); } // namespace ROCKSDB_NAMESPACE diff --git a/dist/linux_amd64/include/rocksdb/file_checksum.h b/dist/linux_amd64/include/rocksdb/file_checksum.h index 37b1744..d79d1c3 100644 --- a/dist/linux_amd64/include/rocksdb/file_checksum.h +++ b/dist/linux_amd64/include/rocksdb/file_checksum.h @@ -76,7 +76,7 @@ class FileChecksumGenFactory { }; // FileChecksumList stores the checksum information of a list of files (e.g., -// SST files). The FileChecksumLIst can be used to store the checksum +// SST files). The FileChecksumList can be used to store the checksum // information of all SST file getting from the MANIFEST, which are // the checksum information of all valid SST file of a DB instance. It can // also be used to store the checksum information of a list of SST files to diff --git a/dist/linux_amd64/include/rocksdb/file_system.h b/dist/linux_amd64/include/rocksdb/file_system.h index e38929d..393b764 100644 --- a/dist/linux_amd64/include/rocksdb/file_system.h +++ b/dist/linux_amd64/include/rocksdb/file_system.h @@ -366,6 +366,10 @@ class FileSystem { return IOStatus::OK(); } +// This seems to clash with a macro on Windows, so #undef it here +#ifdef DeleteFile +#undef DeleteFile +#endif // Delete the named file. virtual IOStatus DeleteFile(const std::string& fname, const IOOptions& options, @@ -1048,7 +1052,8 @@ class FSDirectory { class FileSystemWrapper : public FileSystem { public: // Initialize an EnvWrapper that delegates all calls to *t - explicit FileSystemWrapper(std::shared_ptr t) : target_(t) {} + explicit FileSystemWrapper(const std::shared_ptr& t) + : target_(t) {} ~FileSystemWrapper() override {} const char* Name() const override { return target_->Name(); } diff --git a/dist/linux_amd64/include/rocksdb/utilities/ldb_cmd.h b/dist/linux_amd64/include/rocksdb/utilities/ldb_cmd.h index c7f227f..af0556a 100644 --- a/dist/linux_amd64/include/rocksdb/utilities/ldb_cmd.h +++ b/dist/linux_amd64/include/rocksdb/utilities/ldb_cmd.h @@ -88,6 +88,8 @@ class LDBCommand { virtual void OverrideBaseOptions(); + virtual void OverrideBaseCFOptions(ColumnFamilyOptions* cf_opts); + virtual void SetDBOptions(Options options) { options_ = options; } virtual void SetColumnFamilies( diff --git a/dist/linux_amd64/include/rocksdb/utilities/transaction_db.h b/dist/linux_amd64/include/rocksdb/utilities/transaction_db.h index c1140ec..545e5f1 100644 --- a/dist/linux_amd64/include/rocksdb/utilities/transaction_db.h +++ b/dist/linux_amd64/include/rocksdb/utilities/transaction_db.h @@ -86,7 +86,11 @@ struct RangeDeadlockPath { // RocksDB class RangeLockManagerHandle : public LockManagerHandle { public: - // Total amount of lock memory to use (per column family) + // Set total amount of lock memory to use. + // + // @return 0 Ok + // @return EDOM Failed to set because currently using more memory than + // specified virtual int SetMaxLockMemory(size_t max_lock_memory) = 0; virtual size_t GetMaxLockMemory() = 0; diff --git a/dist/linux_amd64/include/rocksdb/utilities/write_batch_with_index.h b/dist/linux_amd64/include/rocksdb/utilities/write_batch_with_index.h index 424aa15..256638d 100644 --- a/dist/linux_amd64/include/rocksdb/utilities/write_batch_with_index.h +++ b/dist/linux_amd64/include/rocksdb/utilities/write_batch_with_index.h @@ -40,12 +40,13 @@ enum WriteType { kDeleteRangeRecord, kLogDataRecord, kXIDRecord, + kUnknownRecord, }; // an entry for Put, Merge, Delete, or SingleDelete entry for write batches. // Used in WBWIIterator. struct WriteEntry { - WriteType type; + WriteType type = kUnknownRecord; Slice key; Slice value; }; diff --git a/dist/linux_amd64/include/rocksdb/version.h b/dist/linux_amd64/include/rocksdb/version.h index 6a07989..1bb502e 100644 --- a/dist/linux_amd64/include/rocksdb/version.h +++ b/dist/linux_amd64/include/rocksdb/version.h @@ -5,8 +5,8 @@ #pragma once #define ROCKSDB_MAJOR 6 -#define ROCKSDB_MINOR 16 -#define ROCKSDB_PATCH 4 +#define ROCKSDB_MINOR 17 +#define ROCKSDB_PATCH 3 // Do not use these. We made the mistake of declaring macros starting with // double underscore. Now we have to live with our choice. We'll deprecate these diff --git a/dist/linux_amd64/include/rocksdb/write_buffer_manager.h b/dist/linux_amd64/include/rocksdb/write_buffer_manager.h index ae1c98c..c1d1300 100644 --- a/dist/linux_amd64/include/rocksdb/write_buffer_manager.h +++ b/dist/linux_amd64/include/rocksdb/write_buffer_manager.h @@ -43,6 +43,9 @@ class WriteBufferManager { size_t mutable_memtable_memory_usage() const { return memory_active_.load(std::memory_order_relaxed); } + size_t dummy_entries_in_cache_usage() const { + return dummy_size_.load(std::memory_order_relaxed); + } size_t buffer_size() const { return buffer_size_; } // Should only be called from write thread @@ -93,6 +96,7 @@ class WriteBufferManager { std::atomic memory_used_; // Memory that hasn't been scheduled to free. std::atomic memory_active_; + std::atomic dummy_size_; struct CacheRep; std::unique_ptr cache_rep_; diff --git a/dist/linux_amd64/lib/liblz4.a b/dist/linux_amd64/lib/liblz4.a index 2fcefcd..be17e54 100644 Binary files a/dist/linux_amd64/lib/liblz4.a and b/dist/linux_amd64/lib/liblz4.a differ diff --git a/dist/linux_amd64/lib/librocksdb.a b/dist/linux_amd64/lib/librocksdb.a index 9a91086..f832ba8 100644 Binary files a/dist/linux_amd64/lib/librocksdb.a and b/dist/linux_amd64/lib/librocksdb.a differ diff --git a/dist/linux_amd64/lib/libsnappy.a b/dist/linux_amd64/lib/libsnappy.a index d2b055f..413823b 100644 Binary files a/dist/linux_amd64/lib/libsnappy.a and b/dist/linux_amd64/lib/libsnappy.a differ diff --git a/dist/linux_amd64/lib/libz.a b/dist/linux_amd64/lib/libz.a index 7583055..a310f7c 100644 Binary files a/dist/linux_amd64/lib/libz.a and b/dist/linux_amd64/lib/libz.a differ