NO-ISSUE Adapt rocksdb 6.17.3 (#31)

build.sh

@@ -33,7 +33,7 @@ cd $BUILD_PATH && wget https://github.com/facebook/zstd/archive/v${zstd_version}
     -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DZSTD_ZLIB_SUPPORT=ON -DZSTD_LZMA_SUPPORT=OFF -DCMAKE_BUILD_TYPE=Release .. && make -j$(nproc) install && \
     cd $BUILD_PATH && rm -rf * && ldconfig
 
-rocksdb_version="6.16.4"
+rocksdb_version="6.17.3"
 cd $BUILD_PATH && wget https://github.com/facebook/rocksdb/archive/v${rocksdb_version}.tar.gz && tar xzf v${rocksdb_version}.tar.gz && cd rocksdb-${rocksdb_version}/ && \
     mkdir -p build_place && cd build_place && cmake -DCMAKE_BUILD_TYPE=Release $CMAKE_REQUIRED_PARAMS -DCMAKE_PREFIX_PATH=$INSTALL_PREFIX -DWITH_TESTS=OFF -DWITH_GFLAGS=OFF \
     -DWITH_BENCHMARK_TOOLS=OFF -DWITH_TOOLS=OFF -DWITH_MD_LIBRARY=OFF -DWITH_RUNTIME_DEBUG=OFF -DROCKSDB_BUILD_SHARED=OFF -DWITH_SNAPPY=ON -DWITH_LZ4=ON -DWITH_ZLIB=ON \

db.go

@@ -931,10 +931,10 @@ func (db *DB) DropColumnFamily(c *ColumnFamilyHandle) (err error) {
 //
 // The keys counted will begin at Range.Start and end on the key before
 // Range.Limit.
-func (db *DB) GetApproximateSizes(ranges []Range) []uint64 {
+func (db *DB) GetApproximateSizes(ranges []Range) ([]uint64, error) {
 	sizes := make([]uint64, len(ranges))
 	if len(ranges) == 0 {
-		return sizes
+		return sizes, nil
 	}
 
 	cStarts := make([]*C.char, len(ranges))
@@ -948,33 +948,39 @@ func (db *DB) GetApproximateSizes(ranges []Range) []uint64 {
 		cLimitLens[i] = C.size_t(len(r.Limit))
 	}
 
+	var cErr *C.char
 	C.rocksdb_approximate_sizes(
 		db.c,
 		C.int(len(ranges)),
 		&cStarts[0],
 		&cStartLens[0],
 		&cLimits[0],
 		&cLimitLens[0],
-		(*C.uint64_t)(&sizes[0]))
+		(*C.uint64_t)(&sizes[0]),
+		&cErr,
+	)
+
+	// parse error
+	err := fromCError(cErr)
 
 	// free before return
 	for i := range ranges {
 		C.free(unsafe.Pointer(cStarts[i]))
 		C.free(unsafe.Pointer(cLimits[i]))
 	}
 
-	return sizes
+	return sizes, err
 }
 
 // GetApproximateSizesCF returns the approximate number of bytes of file system
 // space used by one or more key ranges in the column family.
 //
 // The keys counted will begin at Range.Start and end on the key before
 // Range.Limit.
-func (db *DB) GetApproximateSizesCF(cf *ColumnFamilyHandle, ranges []Range) []uint64 {
+func (db *DB) GetApproximateSizesCF(cf *ColumnFamilyHandle, ranges []Range) ([]uint64, error) {
 	sizes := make([]uint64, len(ranges))
 	if len(ranges) == 0 {
-		return sizes
+		return sizes, nil
 	}
 
 	cStarts := make([]*C.char, len(ranges))
@@ -988,6 +994,7 @@ func (db *DB) GetApproximateSizesCF(cf *ColumnFamilyHandle, ranges []Range) []ui
 		cLimitLens[i] = C.size_t(len(r.Limit))
 	}
 
+	var cErr *C.char
 	C.rocksdb_approximate_sizes_cf(
 		db.c,
 		cf.c,
@@ -996,15 +1003,20 @@ func (db *DB) GetApproximateSizesCF(cf *ColumnFamilyHandle, ranges []Range) []ui
 		&cStartLens[0],
 		&cLimits[0],
 		&cLimitLens[0],
-		(*C.uint64_t)(&sizes[0]))
+		(*C.uint64_t)(&sizes[0]),
+		&cErr,
+	)
+
+	// parse error
+	err := fromCError(cErr)
 
 	// free before return
 	for i := range ranges {
 		C.free(unsafe.Pointer(cStarts[i]))
 		C.free(unsafe.Pointer(cLimits[i]))
 	}
 
-	return sizes
+	return sizes, err
 }
 
 // SetOptions dynamically changes options through the SetOptions API.

db_test.go

@@ -263,16 +263,19 @@ func TestDBGetApproximateSizes(t *testing.T) {
 	defer db.Close()
 
 	// no ranges
-	sizes := db.GetApproximateSizes(nil)
+	sizes, err := db.GetApproximateSizes(nil)
 	require.EqualValues(t, len(sizes), 0)
+	require.NoError(t, err)
 
 	// range will nil start and limit
-	sizes = db.GetApproximateSizes([]Range{{Start: nil, Limit: nil}})
+	sizes, err = db.GetApproximateSizes([]Range{{Start: nil, Limit: nil}})
 	require.EqualValues(t, sizes, []uint64{0})
+	require.NoError(t, err)
 
 	// valid range
-	sizes = db.GetApproximateSizes([]Range{{Start: []byte{0x00}, Limit: []byte{0xFF}}})
+	sizes, err = db.GetApproximateSizes([]Range{{Start: []byte{0x00}, Limit: []byte{0xFF}}})
 	require.EqualValues(t, sizes, []uint64{0})
+	require.NoError(t, err)
 }
 
 func TestDBGetApproximateSizesCF(t *testing.T) {
@@ -285,14 +288,17 @@ func TestDBGetApproximateSizesCF(t *testing.T) {
 	require.Nil(t, err)
 
 	// no ranges
-	sizes := db.GetApproximateSizesCF(cf, nil)
+	sizes, err := db.GetApproximateSizesCF(cf, nil)
 	require.EqualValues(t, len(sizes), 0)
+	require.NoError(t, err)
 
 	// range will nil start and limit
-	sizes = db.GetApproximateSizesCF(cf, []Range{{Start: nil, Limit: nil}})
+	sizes, err = db.GetApproximateSizesCF(cf, []Range{{Start: nil, Limit: nil}})
 	require.EqualValues(t, sizes, []uint64{0})
+	require.NoError(t, err)
 
 	// valid range
-	sizes = db.GetApproximateSizesCF(cf, []Range{{Start: []byte{0x00}, Limit: []byte{0xFF}}})
+	sizes, err = db.GetApproximateSizesCF(cf, []Range{{Start: []byte{0x00}, Limit: []byte{0xFF}}})
 	require.EqualValues(t, sizes, []uint64{0})
+	require.NoError(t, err)
 }

dist/linux_amd64/include/rocksdb/c.h

@@ -496,13 +496,13 @@ extern ROCKSDB_LIBRARY_API char* rocksdb_property_value_cf(
 extern ROCKSDB_LIBRARY_API void rocksdb_approximate_sizes(
     rocksdb_t* db, int num_ranges, const char* const* range_start_key,
     const size_t* range_start_key_len, const char* const* range_limit_key,
-    const size_t* range_limit_key_len, uint64_t* sizes);
+    const size_t* range_limit_key_len, uint64_t* sizes, char** errptr);
 
 extern ROCKSDB_LIBRARY_API void rocksdb_approximate_sizes_cf(
     rocksdb_t* db, rocksdb_column_family_handle_t* column_family,
     int num_ranges, const char* const* range_start_key,
     const size_t* range_start_key_len, const char* const* range_limit_key,
-    const size_t* range_limit_key_len, uint64_t* sizes);
+    const size_t* range_limit_key_len, uint64_t* sizes, char** errptr);
 
 extern ROCKSDB_LIBRARY_API void rocksdb_compact_range(rocksdb_t* db,
                                                       const char* start_key,
@@ -1981,7 +1981,7 @@ extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get_for_update(
     const char* key, size_t klen, size_t* vlen, unsigned char exclusive,
     char** errptr);
 
-char* rocksdb_transaction_get_for_update_cf(
+extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get_for_update_cf(
     rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options,
     rocksdb_column_family_handle_t* column_family, const char* key, size_t klen,
     size_t* vlen, unsigned char exclusive, char** errptr);

dist/linux_amd64/include/rocksdb/compaction_filter.h

@@ -83,7 +83,7 @@ class CompactionFilter {
   //
   // Note that RocksDB snapshots (i.e. call GetSnapshot() API on a
   // DB* object) will not guarantee to preserve the state of the DB with
-  // CompactionFilter. Data seen from a snapshot might disppear after a
+  // CompactionFilter. Data seen from a snapshot might disappear after a
   // compaction finishes. If you use snapshots, think twice about whether you
   // want to use compaction filter and whether you are using it in a safe way.
   //
@@ -188,7 +188,7 @@ class CompactionFilter {
 
   // This function is deprecated. Snapshots will always be ignored for
   // compaction filters, because we realized that not ignoring snapshots doesn't
-  // provide the gurantee we initially thought it would provide. Repeatable
+  // provide the guarantee we initially thought it would provide. Repeatable
   // reads will not be guaranteed anyway. If you override the function and
   // returns false, we will fail the compaction.
   virtual bool IgnoreSnapshots() const { return true; }

dist/linux_amd64/include/rocksdb/configurable.h

@@ -240,7 +240,7 @@ class Configurable {
   // @param config_options Controls how the object is prepared.  Also contains
   //      a Logger and Env that can be used to initialize this object.
   // @return OK If the object was successfully initialized.
-  // @return InvalidArgument If this object could not be successfull
+  // @return InvalidArgument If this object could not be successfully
   // initialized.
   virtual Status PrepareOptions(const ConfigOptions& config_options);
 

dist/linux_amd64/include/rocksdb/db.h

@@ -1027,20 +1027,22 @@ class DB {
   // Simpler versions of the GetApproximateSizes() method above.
   // The include_flags argumenbt must of type DB::SizeApproximationFlags
   // and can not be NONE.
-  virtual void GetApproximateSizes(ColumnFamilyHandle* column_family,
-                                   const Range* ranges, int n, uint64_t* sizes,
-                                   uint8_t include_flags = INCLUDE_FILES) {
+  virtual Status GetApproximateSizes(ColumnFamilyHandle* column_family,
+                                     const Range* ranges, int n,
+                                     uint64_t* sizes,
+                                     uint8_t include_flags = INCLUDE_FILES) {
     SizeApproximationOptions options;
     options.include_memtabtles =
         (include_flags & SizeApproximationFlags::INCLUDE_MEMTABLES) != 0;
     options.include_files =
         (include_flags & SizeApproximationFlags::INCLUDE_FILES) != 0;
-    Status s = GetApproximateSizes(options, column_family, ranges, n, sizes);
-    s.PermitUncheckedError();
+    return GetApproximateSizes(options, column_family, ranges, n, sizes);
   }
-  virtual void GetApproximateSizes(const Range* ranges, int n, uint64_t* sizes,
-                                   uint8_t include_flags = INCLUDE_FILES) {
-    GetApproximateSizes(DefaultColumnFamily(), ranges, n, sizes, include_flags);
+  virtual Status GetApproximateSizes(const Range* ranges, int n,
+                                     uint64_t* sizes,
+                                     uint8_t include_flags = INCLUDE_FILES) {
+    return GetApproximateSizes(DefaultColumnFamily(), ranges, n, sizes,
+                               include_flags);
   }
 
   // The method is similar to GetApproximateSizes, except it

dist/linux_amd64/include/rocksdb/env.h

@@ -30,6 +30,7 @@
 // Windows API macro interference
 #undef DeleteFile
 #undef GetCurrentTime
+#undef LoadLibrary
 #endif
 
 #if defined(__GNUC__) || defined(__clang__)
@@ -282,7 +283,8 @@ class Env {
   virtual Status FileExists(const std::string& fname) = 0;
 
   // Store in *result the names of the children of the specified directory.
-  // The names are relative to "dir".
+  // The names are relative to "dir", and shall never include the
+  // names `.` or `..`.
   // Original contents of *results are dropped.
   // Returns OK if "dir" exists and "*result" contains its children.
   //         NotFound if "dir" does not exist, the calling process does not have
@@ -295,7 +297,8 @@ class Env {
   // In case the implementation lists the directory prior to iterating the files
   // and files are concurrently deleted, the deleted files will be omitted from
   // result.
-  // The name attributes are relative to "dir".
+  // The name attributes are relative to "dir", and shall never include the
+  // names `.` or `..`.
   // Original contents of *results are dropped.
   // Returns OK if "dir" exists and "*result" contains its children.
   //         NotFound if "dir" does not exist, the calling process does not have
@@ -1663,6 +1666,6 @@ Env* NewTimedEnv(Env* base_env);
 Status NewEnvLogger(const std::string& fname, Env* env,
                     std::shared_ptr<Logger>* result);
 
-std::unique_ptr<Env> NewCompositeEnv(std::shared_ptr<FileSystem> fs);
+std::unique_ptr<Env> NewCompositeEnv(const std::shared_ptr<FileSystem>& fs);
 
 }  // namespace ROCKSDB_NAMESPACE

dist/linux_amd64/include/rocksdb/file_checksum.h

@@ -76,7 +76,7 @@ class FileChecksumGenFactory {
 };
 
 // FileChecksumList stores the checksum information of a list of files (e.g.,
-// SST files). The FileChecksumLIst can be used to store the checksum
+// SST files). The FileChecksumList can be used to store the checksum
 // information of all SST file getting  from the MANIFEST, which are
 // the checksum information of all valid SST file of a DB instance. It can
 // also be used to store the checksum information of a list of SST files to

dist/linux_amd64/include/rocksdb/file_system.h

@@ -366,6 +366,10 @@ class FileSystem {
     return IOStatus::OK();
   }
 
+// This seems to clash with a macro on Windows, so #undef it here
+#ifdef DeleteFile
+#undef DeleteFile
+#endif
   // Delete the named file.
   virtual IOStatus DeleteFile(const std::string& fname,
                               const IOOptions& options,
@@ -1048,7 +1052,8 @@ class FSDirectory {
 class FileSystemWrapper : public FileSystem {
  public:
   // Initialize an EnvWrapper that delegates all calls to *t
-  explicit FileSystemWrapper(std::shared_ptr<FileSystem> t) : target_(t) {}
+  explicit FileSystemWrapper(const std::shared_ptr<FileSystem>& t)
+      : target_(t) {}
   ~FileSystemWrapper() override {}
 
   const char* Name() const override { return target_->Name(); }

dist/linux_amd64/include/rocksdb/utilities/ldb_cmd.h

@@ -88,6 +88,8 @@ class LDBCommand {
 
   virtual void OverrideBaseOptions();
 
+  virtual void OverrideBaseCFOptions(ColumnFamilyOptions* cf_opts);
+
   virtual void SetDBOptions(Options options) { options_ = options; }
 
   virtual void SetColumnFamilies(

dist/linux_amd64/include/rocksdb/utilities/transaction_db.h

@@ -86,7 +86,11 @@ struct RangeDeadlockPath {
 // RocksDB
 class RangeLockManagerHandle : public LockManagerHandle {
  public:
-  // Total amount of lock memory to use (per column family)
+  // Set total amount of lock memory to use.
+  //
+  //  @return 0 Ok
+  //  @return EDOM Failed to set because currently using more memory than
+  //        specified
   virtual int SetMaxLockMemory(size_t max_lock_memory) = 0;
   virtual size_t GetMaxLockMemory() = 0;
 

dist/linux_amd64/include/rocksdb/utilities/write_batch_with_index.h

@@ -40,12 +40,13 @@ enum WriteType {
   kDeleteRangeRecord,
   kLogDataRecord,
   kXIDRecord,
+  kUnknownRecord,
 };
 
 // an entry for Put, Merge, Delete, or SingleDelete entry for write batches.
 // Used in WBWIIterator.
 struct WriteEntry {
-  WriteType type;
+  WriteType type = kUnknownRecord;
   Slice key;
   Slice value;
 };

dist/linux_amd64/include/rocksdb/version.h

@@ -5,8 +5,8 @@
 #pragma once
 
 #define ROCKSDB_MAJOR 6
-#define ROCKSDB_MINOR 16
-#define ROCKSDB_PATCH 4
+#define ROCKSDB_MINOR 17
+#define ROCKSDB_PATCH 3
 
 // Do not use these. We made the mistake of declaring macros starting with
 // double underscore. Now we have to live with our choice. We'll deprecate these

dist/linux_amd64/include/rocksdb/write_buffer_manager.h

@@ -43,6 +43,9 @@ class WriteBufferManager {
   size_t mutable_memtable_memory_usage() const {
     return memory_active_.load(std::memory_order_relaxed);
   }
+  size_t dummy_entries_in_cache_usage() const {
+    return dummy_size_.load(std::memory_order_relaxed);
+  }
   size_t buffer_size() const { return buffer_size_; }
 
   // Should only be called from write thread
@@ -93,6 +96,7 @@ class WriteBufferManager {
   std::atomic<size_t> memory_used_;
   // Memory that hasn't been scheduled to free.
   std::atomic<size_t> memory_active_;
+  std::atomic<size_t> dummy_size_;
   struct CacheRep;
   std::unique_ptr<CacheRep> cache_rep_;