options: reclassify (#51)

Several options were mistakenly classified as db options when in fact
they are column family options - the C interface does not make this
distinction and instead puts them all under one umbrella.

This PR updates options to use nim setter/getter properties allowing
both read and write of most options - lots of options are exposed both
for reading and writing, but hey, one could always add _even more_ of
them - there's certainly no lack!

rocksdb/columnfamily/cfhandle.nim

@@ -28,22 +28,6 @@ proc cPtr*(handle: ColFamilyHandleRef): ColFamilyHandlePtr =
   doAssert not handle.isClosed()
   handle.cPtr
 
-# TODO: These procs below will not work unless using the latest version of rocksdb
-# Currently, when installing librocksdb-dev on linux the RocksDb version used is 6.11.4
-# Need to complete this task: https://github.com/status-im/nim-rocksdb/issues/10
-
-# proc getId*(handle: ColFamilyHandleRef): int =
-#   doAssert not handle.isClosed()
-#   rocksdb_column_family_handle_get_id(handle.cPtr).int
-
-# proc getName*(handle: ColFamilyHandleRef): string =
-#   doAssert not handle.isClosed()
-#   var nameLen: csize_t
-#   $rocksdb_column_family_handle_get_name(handle.cPtr, nameLen.addr)
-
-# proc isDefault*(handle: ColFamilyHandleRef): bool {.inline.} =
-#   handle.getName() == DEFAULT_COLUMN_FAMILY_NAME
-
 proc close*(handle: ColFamilyHandleRef) =
   if not handle.isClosed():
     rocksdb_column_family_handle_destroy(handle.cPtr)

rocksdb/columnfamily/cfopts.nim

@@ -10,12 +10,19 @@
 {.push raises: [].}
 
 import
-  ../lib/librocksdb
+  ../lib/librocksdb, ../options/tableopts
 
 type
+  SlicetransformPtr* = ptr rocksdb_slicetransform_t
+  SlicetransformRef* = ref object
+    cPtr: SlicetransformPtr
+
   ColFamilyOptionsPtr* = ptr rocksdb_options_t
 
   ColFamilyOptionsRef* = ref object
+    # In the C API, both family and database options are exposed using the same
+    # type - CF options are a subset of rocksdb_options_t - when in doubt, check:
+    # https://github.com/facebook/rocksdb/blob/b8c9a2576af6a1d0ffcfbb517dfcb7e7037bd460/include/rocksdb/options.h#L66
     cPtr: ColFamilyOptionsPtr
 
   Compression* {.pure.} = enum
@@ -29,6 +36,21 @@ type
     xpressCompression = rocksdb_xpress_compression
     zstdCompression = rocksdb_zstd_compression
 
+proc createFixedPrefix*(value: int): SlicetransformRef =
+  SlicetransformRef(cPtr: rocksdb_slicetransform_create_fixed_prefix(value.csize_t))
+
+proc isClosed*(s: SlicetransformRef): bool {.inline.} =
+  s.cPtr.isNil()
+
+proc cPtr*(s: SlicetransformRef): SlicetransformPtr =
+  doAssert not s.isClosed()
+  s.cPtr
+
+proc close*(s: SlicetransformRef) =
+  if not s.isClosed():
+    rocksdb_slicetransform_destroy(s.cPtr)
+    s.cPtr = nil
+
 proc newColFamilyOptions*(): ColFamilyOptionsRef =
   ColFamilyOptionsRef(cPtr: rocksdb_options_create())
 
@@ -39,72 +61,94 @@ proc cPtr*(cfOpts: ColFamilyOptionsRef): ColFamilyOptionsPtr =
   doAssert not cfOpts.isClosed()
   cfOpts.cPtr
 
-proc setCreateMissingColumnFamilies*(cfOpts: ColFamilyOptionsRef, flag: bool) =
-  doAssert not cfOpts.isClosed()
-  rocksdb_options_set_create_missing_column_families(cfOpts.cPtr, flag.uint8)
-
-proc defaultColFamilyOptions*(): ColFamilyOptionsRef =
-  let opts = newColFamilyOptions()
+proc close*(cfOpts: ColFamilyOptionsRef) =
+  if not cfOpts.isClosed():
+    rocksdb_options_destroy(cfOpts.cPtr)
+    cfOpts.cPtr = nil
 
-  # rocksdb_options_set_compression(opts.cPtr, rocksdb_lz4_compression)
-  # rocksdb_options_set_bottommost_compression(opts.cPtr, rocksdb_zstd_compression)
+template opt(nname, ntyp, ctyp: untyped) =
+  proc `nname=`*(cfOpts: ColFamilyOptionsRef, value: ntyp) =
+    doAssert not cfOpts.isClosed
+    `rocksdb_options_set nname`(cfOpts.cPtr, value.ctyp)
+
+  proc `nname`*(cfOpts: ColFamilyOptionsRef): ntyp =
+    doAssert not cfOpts.isClosed
+    ntyp `rocksdb_options_get nname`(cfOpts.cPtr)
+
+opt writeBufferSize, int, csize_t
+opt compression, Compression, cint
+opt bottommostCompression, Compression, cint
+opt level0FileNumCompactionTrigger, int, cint
+opt maxBytesForLevelBase, int, uint64
+opt disableAutoCompactions, bool, cint
+
+opt maxWriteBufferNumber, int, cint
+opt minWriteBufferNumberToMerge, int, cint
+opt maxWriteBufferSizeToMaintain, int, int64
+opt inplaceUpdateSupport, bool, uint8
+opt inplaceUpdateNumLocks, int, csize_t
+opt memtablePrefixBloomSizeRatio, float, cdouble
+opt memtableHugePageSize, int, csize_t
+opt bloomLocality, int, uint32
+opt arenaBlockSize, int, csize_t
+opt numLevels, int, cint
+opt level0SlowdownWritesTrigger, int, cint
+opt level0StopWritesTrigger, int, cint
+opt targetFileSizeBase, int, uint64
+opt targetFileSizeMultiplier, int, cint
+opt maxBytesForLevelMultiplier, float, cdouble
+opt maxCompactionBytes, int, uint64
+opt softPendingCompactionBytesLimit, int, csize_t
+opt hardPendingCompactionBytesLimit, int, csize_t
+opt maxSequentialSkipInIterations, int, uint64
+opt maxSuccessiveMerges, int, csize_t
+opt optimizeFiltersForHits, bool, cint
+opt paranoidChecks, bool, uint8
+opt reportBgIoStats, bool, cint
+opt enableBlobFiles, bool, uint8
+opt minBlobSize, int, uint64
+opt blobFileSize, int, uint64
+opt blobCompressionType, Compression, cint
+opt enableBlobGC, bool, uint8
+opt blobGCAgeCutoff, float, cdouble
+opt blobGCForceThreshold, float, cdouble
+opt blobCompactionReadaheadSize, int, uint64
+opt blobFileStartingLevel, int, cint
 
-  # Enable creating column families if they do not exist
-  opts.setCreateMissingColumnFamilies(true)
-  return opts
+proc defaultColFamilyOptions*(): ColFamilyOptionsRef =
+  newColFamilyOptions()
 
-# TODO: These procs below will not work unless using the latest version of rocksdb
-# Currently, when installing librocksdb-dev on linux the RocksDb version used is 6.11.4
-# Need to complete this task: https://github.com/status-im/nim-rocksdb/issues/10
+# proc setFixedPrefixExtractor*(dbOpts: ColFamilyOptionsRef, length: int) =
+#   doAssert not dbOpts.isClosed()
+#   rocksdb_options_set_prefix_extractor(
+#     dbOpts.cPtr, rocksdb_slicetransform_create_fixed_prefix(length.csize_t))
 
-# proc getCreateMissingColumnFamilies*(cfOpts: ColFamilyOptionsRef): bool =
-#   doAssert not cfOpts.isClosed()
-#   rocksdb_options_get_create_missing_column_families(cfOpts.cPtr).bool
+proc `setPrefixExtractor`*(cfOpts: ColFamilyOptionsRef, value: SlicetransformRef) =
+  doAssert not cfOpts.isClosed()
+  rocksdb_options_set_prefix_extractor(cfOpts.cPtr, value.cPtr)
 
-proc setWriteBufferSize*(dbOpts: ColFamilyOptionsRef, maxBufferSize: int) =
-  doAssert not dbOpts.isClosed()
-  rocksdb_options_set_write_buffer_size(dbOpts.cPtr, maxBufferSize.csize_t)
+proc `blockBasedTableFactory=`*(cfOpts: ColFamilyOptionsRef, tableOpts: TableOptionsRef) =
+  doAssert not cfOpts.isClosed()
+  rocksdb_options_set_block_based_table_factory(cfOpts.cPtr, tableOpts.cPtr)
 
 # https://github.com/facebook/rocksdb/wiki/MemTable
 proc setHashSkipListRep*(
-    dbOpts: ColFamilyOptionsRef, bucketCount, skipListHeight,
+    cfOpts: ColFamilyOptionsRef, bucketCount, skipListHeight,
     skipListBranchingFactor: int) =
-  doAssert not dbOpts.isClosed()
+  doAssert not cfOpts.isClosed()
   rocksdb_options_set_hash_skip_list_rep(
-    dbOpts.cPtr, bucketCount.csize_t, skipListHeight.cint,
+    cfOpts.cPtr, bucketCount.csize_t, skipListHeight.cint,
     skipListBranchingFactor.cint)
 
 proc setHashLinkListRep*(
-    dbOpts: ColFamilyOptionsRef, bucketCount: int) =
-  doAssert not dbOpts.isClosed()
-  rocksdb_options_set_hash_link_list_rep(dbOpts.cPtr, bucketCount.csize_t)
+    cfOpts: ColFamilyOptionsRef, bucketCount: int) =
+  doAssert not cfOpts.isClosed()
+  rocksdb_options_set_hash_link_list_rep(cfOpts.cPtr, bucketCount.csize_t)
 
-proc setMemtableVectorRep*(dbOpts: ColFamilyOptionsRef) =
-  doAssert not dbOpts.isClosed()
-  rocksdb_options_set_memtable_vector_rep(dbOpts.cPtr)
+proc setMemtableVectorRep*(cfOpts: ColFamilyOptionsRef) =
+  doAssert not cfOpts.isClosed()
+  rocksdb_options_set_memtable_vector_rep(cfOpts.cPtr)
 
-proc setMemtableWholeKeyFiltering*(dbOpts: ColFamilyOptionsRef, value: bool) =
+proc `memtableWholeKeyFiltering=`*(dbOpts: ColFamilyOptionsRef, value: bool) =
   doAssert not dbOpts.isClosed()
   rocksdb_options_set_memtable_whole_key_filtering(dbOpts.cPtr, value.uint8)
-
-proc setMemtablePrefixBloomSizeRatio*(dbOpts: ColFamilyOptionsRef, value: float) =
-  doAssert not dbOpts.isClosed()
-  rocksdb_options_set_memtable_prefix_bloom_size_ratio(dbOpts.cPtr, value)
-
-proc setFixedPrefixExtractor*(dbOpts: ColFamilyOptionsRef, length: int) =
-  doAssert not dbOpts.isClosed()
-  rocksdb_options_set_prefix_extractor(
-    dbOpts.cPtr, rocksdb_slicetransform_create_fixed_prefix(length.csize_t))
-
-proc setCompression*(dbOpts: ColFamilyOptionsRef, value: Compression) =
-  doAssert not dbOpts.isClosed()
-  rocksdb_options_set_compression(dbOpts.cPtr, value.cint)
-
-proc setBottommostCompression*(dbOpts: ColFamilyOptionsRef, value: Compression) =
-  doAssert not dbOpts.isClosed()
-  rocksdb_options_set_bottommost_compression(dbOpts.cPtr, value.cint)
-
-proc close*(cfOpts: ColFamilyOptionsRef) =
-  if not cfOpts.isClosed():
-    rocksdb_options_destroy(cfOpts.cPtr)
-    cfOpts.cPtr = nil

rocksdb/options/backupopts.nim

@@ -32,8 +32,6 @@ proc cPtr*(engineOpts: BackupEngineOptionsRef): BackupEngineOptionsPtr =
 
 proc defaultBackupEngineOptions*(): BackupEngineOptionsRef {.inline.} =
   let opts = newBackupEngineOptions()
-  # rocksdb_options_set_compression(opts.cPtr, rocksdb_lz4_compression)
-  # rocksdb_options_set_bottommost_compression(opts.cPtr, rocksdb_zstd_compression)
   opts
 
 

rocksdb/options/dbopts.nim

@@ -32,105 +32,90 @@ proc cPtr*(dbOpts: DbOptionsRef): DbOptionsPtr =
   doAssert not dbOpts.isClosed()
   dbOpts.cPtr
 
-proc setIncreaseParallelism*(dbOpts: DbOptionsRef, totalThreads: int) =
+proc increaseParallelism*(dbOpts: DbOptionsRef, totalThreads: int) =
   doAssert totalThreads > 0
   doAssert not dbOpts.isClosed()
   rocksdb_options_increase_parallelism(dbOpts.cPtr, totalThreads.cint)
 
-proc setCreateIfMissing*(dbOpts: DbOptionsRef, flag: bool) =
-  doAssert not dbOpts.isClosed()
-  rocksdb_options_set_create_if_missing(dbOpts.cPtr, flag.uint8)
-
-proc setMaxOpenFiles*(dbOpts: DbOptionsRef, maxOpenFiles: int) =
-  doAssert maxOpenFiles >= -1
-  doAssert not dbOpts.isClosed()
-  rocksdb_options_set_max_open_files(dbOpts.cPtr, maxOpenFiles.cint)
-
-proc setCreateMissingColumnFamilies*(dbOpts: DbOptionsRef, flag: bool) =
-  doAssert not dbOpts.isClosed()
-  rocksdb_options_set_create_missing_column_families(dbOpts.cPtr, flag.uint8)
-
-proc setWriteBufferSize*(dbOpts: DbOptionsRef, maxBufferSize: int) =
-  doAssert not dbOpts.isClosed()
-  rocksdb_options_set_write_buffer_size(dbOpts.cPtr, maxBufferSize.csize_t)
-
-proc setRowCache*(dbOpts: DbOptionsRef, cache: CacheRef) =
+# Options roughly in the order found in `options.h`
+
+template opt(nname, ntyp, ctyp: untyped) =
+  proc `nname=`*(dbOpts: DbOptionsRef, value: ntyp) =
+    doAssert not dbOpts.isClosed
+    `rocksdb_options_set nname`(dbOpts.cPtr, value.ctyp)
+
+  proc `nname`*(dbOpts: DbOptionsRef): ntyp =
+    doAssert not dbOpts.isClosed
+    ntyp `rocksdb_options_get nname`(dbOpts.cPtr)
+
+opt createIfMissing, bool, uint8
+opt createMissingColumnFamilies, bool, uint8
+opt errorIfExists, bool, uint8
+opt paranoidChecks, bool, uint8
+
+opt maxOpenFiles, int, cint
+opt maxFileOpeningThreads, int, cint
+opt maxTotalWalSize, int, uint64
+opt useFsync, bool, cint
+opt deleteObsoleteFilesPeriodMicros, int, uint64
+opt maxBackgroundJobs, int, cint
+opt maxBackgroundCompactions, int, cint
+opt maxSubcompactions, int, uint32
+opt maxLogFileSize, int, csize_t
+opt logFiletimeToRoll, int, csize_t
+opt keepLogFileNum, int, csize_t
+opt recycleLogFileNum, int, csize_t
+opt maxManifestFileSize, int, csize_t
+opt tableCacheNumshardbits, int, cint
+opt walTtlSeconds, int, uint64
+opt walSizeLimitMB, int, uint64
+opt manifestPreallocationSize, int, csize_t
+opt allowMmapReads, bool, uint8
+opt allowMmapWrites, bool, uint8
+opt useDirectReads, bool, uint8
+opt useDirectIoForFlushAndCompaction, bool, uint8
+opt isFdCloseOnExec, bool, uint8
+opt statsDumpPeriodSec, int, cuint
+opt statsPersistPeriodSec, int, cuint
+opt adviseRandomOnOpen, bool, uint8
+opt dbWriteBufferSize, int, csize_t
+opt writableFileMaxBufferSize, int, csize_t
+opt useAdaptiveMutex, bool, uint8
+opt bytesPerSync, int, uint64
+opt walBytesPerSync, int, uint64
+opt enablePipelinedWrite, bool, uint8
+opt unorderedWrite, bool, uint8
+opt allowConcurrentMemtableWrite, bool, uint8
+opt enableWriteThreadAdaptiveYield, bool, uint8
+opt skipStatsUpdateOnDbOpen, bool, uint8
+opt skipCheckingSstFileSizesOnDbOpen, bool, uint8
+opt allowIngestBehind, bool, uint8
+opt manualWalFlush, bool, uint8
+opt atomicFlush, bool, uint8
+opt avoidUnnecessaryBlockingIo, bool, uint8
+
+proc `rowCache=`*(dbOpts: DbOptionsRef, cache: CacheRef) =
   doAssert not dbOpts.isClosed()
   rocksdb_options_set_row_cache(dbOpts.cPtr, cache.cPtr)
 
-proc setMaxBackgroundJobs*(dbOpts: DbOptionsRef, jobs: int) =
-  doAssert not dbOpts.isClosed()
-  rocksdb_options_set_max_background_jobs(dbOpts.cPtr, jobs.cint)
-
-proc setBytesPerSync*(dbOpts: DbOptionsRef, bytes: int64) =
-  doAssert not dbOpts.isClosed()
-  rocksdb_options_set_bytes_per_sync(dbOpts.cPtr, bytes.csize_t)
-
-proc setBlockBasedTableFactory*(dbOpts: DbOptionsRef, tableOpts: TableOptionsRef) =
-  doAssert not dbOpts.isClosed()
-  rocksdb_options_set_block_based_table_factory(dbOpts.cPtr, tableOpts.cPtr)
-
-proc setTargetFileSizeBase*(dbOpts: DbOptionsRef, bytes: int64) =
-  doAssert not dbOpts.isClosed()
-  rocksdb_options_set_target_file_size_base(dbOpts.cPtr, bytes.csize_t)
-
-proc setMaxBytesForLevelBase*(dbOpts: DbOptionsRef, bytes: int64) =
-  doAssert not dbOpts.isClosed()
-  rocksdb_options_set_max_bytes_for_level_base(dbOpts.cPtr, bytes.csize_t)
-
-proc setMaxBytesForLevelMultiplier*(dbOpts: DbOptionsRef, multiplier: float) =
-  doAssert not dbOpts.isClosed()
-  rocksdb_options_set_max_bytes_for_level_multiplier(dbOpts.cPtr, multiplier.cdouble)
-
-proc setAllowConcurrentMemtableWrite*(dbOpts: DbOptionsRef, value: bool) =
-  doAssert not dbOpts.isClosed()
-  rocksdb_options_set_allow_concurrent_memtable_write(dbOpts.cPtr, value.uint8)
-
-proc setOptimizeFiltersForHits*(dbOpts: DbOptionsRef, value: bool) =
-  doAssert not dbOpts.isClosed()
-  rocksdb_options_set_optimize_filters_for_hits(dbOpts.cPtr, value.cint)
-
-proc setMaxTotalWalSize*(dbOpts: DbOptionsRef, size: int) =
-  doAssert not dbOpts.isClosed()
-  rocksdb_options_set_max_total_wal_size(dbOpts.cPtr, size.csize_t)
-
 proc defaultDbOptions*(): DbOptionsRef =
   let opts: DbOptionsRef = newDbOptions()
 
   # Optimize RocksDB. This is the easiest way to get RocksDB to perform well:
-  opts.setIncreaseParallelism(countProcessors())
-  opts.setCreateIfMissing(true)
+  opts.increaseParallelism(countProcessors())
+  opts.createIfMissing = true
 
-  # default set to keep all files open (-1), allow setting it to a specific
-  # value, e.g. in case the application limit would be reached.
-  opts.setMaxOpenFiles(-1)
   # Enable creating column families if they do not exist
-  opts.setCreateMissingColumnFamilies(true)
+  opts.createMissingColumnFamilies = true
 
   # Options recommended by rocksdb devs themselves, for new databases
   # https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning#other-general-options
 
-  opts.setMaxBackgroundJobs(6)
-  opts.setBytesPerSync(1048576)
+  opts.maxBackgroundJobs = 6
+  opts.bytesPerSync = 1048576
 
   opts
 
-# TODO: These procs below will not work unless using the latest version of rocksdb
-# Currently, when installing librocksdb-dev on linux the RocksDb version used is 6.11.4
-# Need to complete this task: https://github.com/status-im/nim-rocksdb/issues/10
-
-# proc getCreateIfMissing*(dbOpts: DbOptionsRef): bool =
-#   doAssert not dbOpts.isClosed()
-#   rocksdb_options_get_create_if_missing(dbOpts.cPtr).bool
-
-# proc getMaxOpenFiles*(dbOpts: DbOptionsRef): int =
-#   doAssert not dbOpts.isClosed()
-#   rocksdb_options_get_max_open_files(dbOpts.cPtr).int
-
-# proc getCreateMissingColumnFamilies*(dbOpts: DbOptionsRef): bool =
-#   doAssert not dbOpts.isClosed()
-#   rocksdb_options_get_create_missing_column_families(dbOpts.cPtr).bool
-
 proc close*(dbOpts: DbOptionsRef) =
   if not dbOpts.isClosed():
     rocksdb_options_destroy(dbOpts.cPtr)