diff --git a/rocksdb/backup.nim b/rocksdb/backup.nim index b29b20d..7c88bc5 100644 --- a/rocksdb/backup.nim +++ b/rocksdb/backup.nim @@ -25,17 +25,21 @@ type backupOpts: BackupEngineOptionsRef proc openBackupEngine*( - path: string, backupOpts = defaultBackupEngineOptions() + path: string, backupOpts = defaultBackupEngineOptions(autoClose = true) ): RocksDBResult[BackupEngineRef] = ## Create a new backup engine. The `path` parameter is the path of the backup ## directory. Note that the same directory should not be used for both backups ## and the database itself. + ## + ## If no `backupOpts` are provided, the default options will be used. These + ## default backup options will be closed when the backup engine is closed. + ## If `backupOpts` are provided, they will need to be closed manually. var errors: cstring let backupEnginePtr = rocksdb_backup_engine_open( backupOpts.cPtr, path.cstring, cast[cstringArray](errors.addr) ) - bailOnErrors(errors) + bailOnErrors(errors, backupOpts = backupOpts) let engine = BackupEngineRef(cPtr: backupEnginePtr, path: path, backupOpts: backupOpts) @@ -88,3 +92,6 @@ proc close*(backupEngine: BackupEngineRef) = if not backupEngine.isClosed(): rocksdb_backup_engine_close(backupEngine.cPtr) backupEngine.cPtr = nil + + if backupEngine.backupOpts.autoClose: + backupEngine.backupOpts.close() diff --git a/rocksdb/columnfamily/cfdescriptor.nim b/rocksdb/columnfamily/cfdescriptor.nim index 3c40579..c02eb7e 100644 --- a/rocksdb/columnfamily/cfdescriptor.nim +++ b/rocksdb/columnfamily/cfdescriptor.nim @@ -9,16 +9,18 @@ {.push raises: [].} -import ../internal/utils, ./cfopts +import ./cfopts export cfopts +const DEFAULT_COLUMN_FAMILY_NAME* = "default" + type ColFamilyDescriptor* = object name: string options: ColFamilyOptionsRef proc initColFamilyDescriptor*( - name: string, options = defaultColFamilyOptions() + name: string, options: ColFamilyOptionsRef ): ColFamilyDescriptor = ColFamilyDescriptor(name: name, options: options) @@ -28,11 +30,16 @@ proc name*(descriptor: ColFamilyDescriptor): string {.inline.} = proc options*(descriptor: ColFamilyDescriptor): ColFamilyOptionsRef {.inline.} = descriptor.options +proc autoClose*(descriptor: ColFamilyDescriptor): bool {.inline.} = + descriptor.options.autoClose + proc isDefault*(descriptor: ColFamilyDescriptor): bool {.inline.} = descriptor.name == DEFAULT_COLUMN_FAMILY_NAME -proc defaultColFamilyDescriptor*(): ColFamilyDescriptor {.inline.} = - initColFamilyDescriptor(DEFAULT_COLUMN_FAMILY_NAME) +proc defaultColFamilyDescriptor*(autoClose = false): ColFamilyDescriptor {.inline.} = + initColFamilyDescriptor( + DEFAULT_COLUMN_FAMILY_NAME, defaultColFamilyOptions(autoClose = autoClose) + ) proc isClosed*(descriptor: ColFamilyDescriptor): bool {.inline.} = descriptor.options.isClosed() diff --git a/rocksdb/columnfamily/cfopts.nim b/rocksdb/columnfamily/cfopts.nim index cba148d..ae7e237 100644 --- a/rocksdb/columnfamily/cfopts.nim +++ b/rocksdb/columnfamily/cfopts.nim @@ -23,6 +23,7 @@ type # type - CF options are a subset of rocksdb_options_t - when in doubt, check: # https://github.com/facebook/rocksdb/blob/b8c9a2576af6a1d0ffcfbb517dfcb7e7037bd460/include/rocksdb/options.h#L66 cPtr: ColFamilyOptionsPtr + autoClose*: bool # if true then close will be called when the database is closed Compression* {.pure.} = enum # Use a slightly clunky name here to avoid global symbol conflicts @@ -50,8 +51,8 @@ proc close*(s: SlicetransformRef) = rocksdb_slicetransform_destroy(s.cPtr) s.cPtr = nil -proc newColFamilyOptions*(): ColFamilyOptionsRef = - ColFamilyOptionsRef(cPtr: rocksdb_options_create()) +proc newColFamilyOptions*(autoClose = false): ColFamilyOptionsRef = + ColFamilyOptionsRef(cPtr: rocksdb_options_create(), autoClose: autoClose) proc isClosed*(cfOpts: ColFamilyOptionsRef): bool {.inline.} = cfOpts.cPtr.isNil() @@ -114,8 +115,8 @@ opt blobGCForceThreshold, float, cdouble opt blobCompactionReadaheadSize, int, uint64 opt blobFileStartingLevel, int, cint -proc defaultColFamilyOptions*(): ColFamilyOptionsRef = - newColFamilyOptions() +proc defaultColFamilyOptions*(autoClose = false): ColFamilyOptionsRef = + newColFamilyOptions(autoClose) # proc setFixedPrefixExtractor*(dbOpts: ColFamilyOptionsRef, length: int) = # doAssert not dbOpts.isClosed() diff --git a/rocksdb/internal/utils.nim b/rocksdb/internal/utils.nim index cc4a21a..2de5923 100644 --- a/rocksdb/internal/utils.nim +++ b/rocksdb/internal/utils.nim @@ -9,17 +9,42 @@ {.push raises: [].} -import std/locks, ../lib/librocksdb - -const DEFAULT_COLUMN_FAMILY_NAME* = "default" +import + std/locks, + ../lib/librocksdb, + ../options/[dbopts, readopts, writeopts, backupopts], + ../transactions/txdbopts, + ../columnfamily/cfdescriptor proc createLock*(): Lock = var lock = Lock() initLock(lock) lock -template bailOnErrors*(errors: cstring): auto = +template autoCloseNonNil*(opts: typed) = + if not opts.isNil and opts.autoClose: + opts.close() + +template bailOnErrors*( + errors: cstring, + dbOpts: DbOptionsRef = nil, + readOpts: ReadOptionsRef = nil, + writeOpts: WriteOptionsRef = nil, + txDbOpts: TransactionDbOptionsRef = nil, + backupOpts: BackupEngineOptionsRef = nil, + cfDescriptors: openArray[ColFamilyDescriptor] = @[], +): auto = if not errors.isNil: + autoCloseNonNil(dbOpts) + autoCloseNonNil(readOpts) + autoCloseNonNil(writeOpts) + autoCloseNonNil(txDbOpts) + autoCloseNonNil(backupOpts) + + for cfDesc in cfDescriptors: + if cfDesc.autoClose: + cfDesc.close() + let res = err($(errors)) rocksdb_free(errors) return res diff --git a/rocksdb/options/backupopts.nim b/rocksdb/options/backupopts.nim index 769a2ac..75debb6 100644 --- a/rocksdb/options/backupopts.nim +++ b/rocksdb/options/backupopts.nim @@ -16,9 +16,10 @@ type BackupEngineOptionsRef* = ref object cPtr: BackupEngineOptionsPtr + autoClose*: bool # if true then close will be called when the backup engine is closed -proc newBackupEngineOptions*(): BackupEngineOptionsRef = - BackupEngineOptionsRef(cPtr: rocksdb_options_create()) +proc newBackupEngineOptions*(autoClose = false): BackupEngineOptionsRef = + BackupEngineOptionsRef(cPtr: rocksdb_options_create(), autoClose: autoClose) proc isClosed*(engineOpts: BackupEngineOptionsRef): bool {.inline.} = engineOpts.cPtr.isNil() @@ -29,8 +30,11 @@ proc cPtr*(engineOpts: BackupEngineOptionsRef): BackupEngineOptionsPtr = # TODO: Add setters and getters for backup options properties. -proc defaultBackupEngineOptions*(): BackupEngineOptionsRef {.inline.} = - let opts = newBackupEngineOptions() +proc defaultBackupEngineOptions*(autoClose = false): BackupEngineOptionsRef {.inline.} = + let opts = newBackupEngineOptions(autoClose) + + # TODO: set defaults here + opts proc close*(engineOpts: BackupEngineOptionsRef) = diff --git a/rocksdb/options/dbopts.nim b/rocksdb/options/dbopts.nim index 28ba3bb..e3f9c0f 100644 --- a/rocksdb/options/dbopts.nim +++ b/rocksdb/options/dbopts.nim @@ -18,9 +18,10 @@ type DbOptionsRef* = ref object cPtr: DbOptionsPtr + autoClose*: bool # if true then close will be called when the database is closed -proc newDbOptions*(): DbOptionsRef = - DbOptionsRef(cPtr: rocksdb_options_create()) +proc newDbOptions*(autoClose = false): DbOptionsRef = + DbOptionsRef(cPtr: rocksdb_options_create(), autoClose: autoClose) proc isClosed*(dbOpts: DbOptionsRef): bool {.inline.} = dbOpts.cPtr.isNil() @@ -95,8 +96,8 @@ proc `rowCache=`*(dbOpts: DbOptionsRef, cache: CacheRef) = doAssert not dbOpts.isClosed() rocksdb_options_set_row_cache(dbOpts.cPtr, cache.cPtr) -proc defaultDbOptions*(): DbOptionsRef = - let opts: DbOptionsRef = newDbOptions() +proc defaultDbOptions*(autoClose = false): DbOptionsRef = + let opts: DbOptionsRef = newDbOptions(autoClose) # Optimize RocksDB. This is the easiest way to get RocksDB to perform well: opts.increaseParallelism(countProcessors()) diff --git a/rocksdb/options/readopts.nim b/rocksdb/options/readopts.nim index 0e719c8..fc15960 100644 --- a/rocksdb/options/readopts.nim +++ b/rocksdb/options/readopts.nim @@ -16,9 +16,10 @@ type ReadOptionsRef* = ref object cPtr: ReadOptionsPtr + autoClose*: bool # if true then close will be called when the database is closed -proc newReadOptions*(): ReadOptionsRef = - ReadOptionsRef(cPtr: rocksdb_readoptions_create()) +proc newReadOptions*(autoClose = false): ReadOptionsRef = + ReadOptionsRef(cPtr: rocksdb_readoptions_create(), autoClose: autoClose) proc isClosed*(readOpts: ReadOptionsRef): bool {.inline.} = readOpts.cPtr.isNil() @@ -29,8 +30,8 @@ proc cPtr*(readOpts: ReadOptionsRef): ReadOptionsPtr = # TODO: Add setters and getters for read options properties. -proc defaultReadOptions*(): ReadOptionsRef {.inline.} = - newReadOptions() +proc defaultReadOptions*(autoClose = false): ReadOptionsRef {.inline.} = + newReadOptions(autoClose) # TODO: set prefered defaults proc close*(readOpts: ReadOptionsRef) = diff --git a/rocksdb/options/writeopts.nim b/rocksdb/options/writeopts.nim index 9648f06..3702ed3 100644 --- a/rocksdb/options/writeopts.nim +++ b/rocksdb/options/writeopts.nim @@ -16,9 +16,10 @@ type WriteOptionsRef* = ref object cPtr: WriteOptionsPtr + autoClose*: bool # if true then close will be called when the database is closed -proc newWriteOptions*(): WriteOptionsRef = - WriteOptionsRef(cPtr: rocksdb_writeoptions_create()) +proc newWriteOptions*(autoClose = false): WriteOptionsRef = + WriteOptionsRef(cPtr: rocksdb_writeoptions_create(), autoClose: autoClose) proc isClosed*(writeOpts: WriteOptionsRef): bool {.inline.} = writeOpts.cPtr.isNil() @@ -29,8 +30,8 @@ proc cPtr*(writeOpts: WriteOptionsRef): WriteOptionsPtr = # TODO: Add setters and getters for write options properties. -proc defaultWriteOptions*(): WriteOptionsRef {.inline.} = - newWriteOptions() +proc defaultWriteOptions*(autoClose = false): WriteOptionsRef {.inline.} = + newWriteOptions(autoClose) # TODO: set prefered defaults proc close*(writeOpts: WriteOptionsRef) = diff --git a/rocksdb/rocksdb.nim b/rocksdb/rocksdb.nim index d012494..a5443d7 100644 --- a/rocksdb/rocksdb.nim +++ b/rocksdb/rocksdb.nim @@ -47,6 +47,7 @@ type path: string dbOpts: DbOptionsRef readOpts: ReadOptionsRef + cfDescriptors: seq[ColFamilyDescriptor] defaultCfHandle: ColFamilyHandleRef cfTable: ColFamilyTableRef @@ -56,9 +57,7 @@ type writeOpts: WriteOptionsRef ingestOptsPtr: IngestExternalFilesOptionsPtr -proc listColumnFamilies*( - path: string, dbOpts = DbOptionsRef(nil) -): RocksDBResult[seq[string]] = +proc listColumnFamilies*(path: string): RocksDBResult[seq[string]] = ## List exisiting column families on disk. This might be used to find out ## whether there were some columns missing with the version on disk. ## @@ -69,59 +68,50 @@ proc listColumnFamilies*( ## Note that the on-the-fly adding might not be needed in the way described ## above once rocksdb has been upgraded to the latest version, see comments ## at the end of ./columnfamily/cfhandle.nim. - ## - let useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts) - defer: - if dbOpts.isNil: - useDbOpts.close() var - lencf: csize_t + cfLen: csize_t errors: cstring - let cList = rocksdb_list_column_families( - useDbOpts.cPtr, path.cstring, addr lencf, cast[cstringArray](errors.addr) - ) - bailOnErrors(errors) + let + dbOpts = defaultDbOptions(autoClose = true) + cfList = rocksdb_list_column_families( + dbOpts.cPtr, path.cstring, addr cfLen, cast[cstringArray](errors.addr) + ) + bailOnErrors(errors, dbOpts) - var cfs: seq[string] - if not cList.isNil: - defer: - rocksdb_free(cList) + if cfList.isNil or cfLen == 0: + return ok(newSeqOfCap[string](0)) - for n in 0 ..< lencf: - if cList[n].isNil: - # Clean up the rest - for z in n + 1 ..< lencf: - if not cList[z].isNil: - rocksdb_free(cList[z]) - return err("short reply") + defer: + rocksdb_list_column_families_destroy(cfList, cfLen) + dbOpts.close() - cfs.add $cList[n] - rocksdb_free(cList[n]) + var colFamilyNames = newSeqOfCap[string](cfLen) + for i in 0 ..< cfLen: + colFamilyNames.add($cfList[i]) - ok cfs + ok(colFamilyNames) proc openRocksDb*( path: string, - dbOpts = DbOptionsRef(nil), - readOpts = ReadOptionsRef(nil), - writeOpts = WriteOptionsRef(nil), + dbOpts = defaultDbOptions(autoClose = true), + readOpts = defaultReadOptions(autoClose = true), + writeOpts = defaultWriteOptions(autoClose = true), columnFamilies: openArray[ColFamilyDescriptor] = [], ): RocksDBResult[RocksDbReadWriteRef] = ## Open a RocksDB instance in read-write mode. If `columnFamilies` is empty ## then it will open the default column family. If `dbOpts`, `readOpts`, or ## `writeOpts` are not supplied then the default options will be used. + ## These default options will be closed when the database is closed. + ## If any options are provided, they will need to be closed manually. + ## ## By default, column families will be created if they don't yet exist. ## All existing column families must be specified if the database has ## previously created any column families. - let useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts) - defer: - if dbOpts.isNil: - useDbOpts.close() var cfs = columnFamilies.toSeq() if DEFAULT_COLUMN_FAMILY_NAME notin columnFamilies.mapIt(it.name()): - cfs.add(defaultColFamilyDescriptor()) + cfs.add(defaultColFamilyDescriptor(autoClose = true)) var cfNames = cfs.mapIt(it.name().cstring) @@ -129,7 +119,7 @@ proc openRocksDb*( cfHandles = newSeq[ColFamilyHandlePtr](cfs.len) errors: cstring let rocksDbPtr = rocksdb_open_column_families( - useDbOpts.cPtr, + dbOpts.cPtr, path.cstring, cfNames.len().cint, cast[cstringArray](cfNames[0].addr), @@ -137,12 +127,9 @@ proc openRocksDb*( cfHandles[0].addr, cast[cstringArray](errors.addr), ) - bailOnErrors(errors) + bailOnErrors(errors, dbOpts, readOpts, writeOpts, cfDescriptors = cfs) let - dbOpts = useDbOpts # don't close on exit - readOpts = (if readOpts.isNil: defaultReadOptions() else: readOpts) - writeOpts = (if writeOpts.isNil: defaultWriteOptions() else: writeOpts) cfTable = newColFamilyTable(cfNames.mapIt($it), cfHandles) db = RocksDbReadWriteRef( lock: createLock(), @@ -151,6 +138,7 @@ proc openRocksDb*( dbOpts: dbOpts, readOpts: readOpts, writeOpts: writeOpts, + cfDescriptors: cfs, ingestOptsPtr: rocksdb_ingestexternalfileoptions_create(), defaultCfHandle: cfTable.get(DEFAULT_COLUMN_FAMILY_NAME), cfTable: cfTable, @@ -159,25 +147,24 @@ proc openRocksDb*( proc openRocksDbReadOnly*( path: string, - dbOpts = DbOptionsRef(nil), - readOpts = ReadOptionsRef(nil), + dbOpts = defaultDbOptions(autoClose = true), + readOpts = defaultReadOptions(autoClose = true), columnFamilies: openArray[ColFamilyDescriptor] = [], errorIfWalFileExists = false, ): RocksDBResult[RocksDbReadOnlyRef] = ## Open a RocksDB instance in read-only mode. If `columnFamilies` is empty ## then it will open the default column family. If `dbOpts` or `readOpts` are - ## not supplied then the default options will be used. By default, column - ## families will be created if they don't yet exist. If the database already - ## contains any column families, then all or a subset of the existing column - ## families can be opened for reading. - let useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts) - defer: - if dbOpts.isNil: - useDbOpts.close() + ## not supplied then the default options will be used. + ## These default options will be closed when the database is closed. + ## If any options are provided, they will need to be closed manually. + ## + ## By default, column families will be created if they don't yet exist. + ## If the database already contains any column families, then all or + ## a subset of the existing column families can be opened for reading. var cfs = columnFamilies.toSeq() if DEFAULT_COLUMN_FAMILY_NAME notin columnFamilies.mapIt(it.name()): - cfs.add(defaultColFamilyDescriptor()) + cfs.add(defaultColFamilyDescriptor(autoClose = true)) var cfNames = cfs.mapIt(it.name().cstring) @@ -185,7 +172,7 @@ proc openRocksDbReadOnly*( cfHandles = newSeq[ColFamilyHandlePtr](cfs.len) errors: cstring let rocksDbPtr = rocksdb_open_for_read_only_column_families( - useDbOpts.cPtr, + dbOpts.cPtr, path.cstring, cfNames.len().cint, cast[cstringArray](cfNames[0].addr), @@ -194,11 +181,9 @@ proc openRocksDbReadOnly*( errorIfWalFileExists.uint8, cast[cstringArray](errors.addr), ) - bailOnErrors(errors) + bailOnErrors(errors, dbOpts, readOpts, cfDescriptors = cfs) let - dbOpts = useDbOpts # don't close on exit - readOpts = (if readOpts.isNil: defaultReadOptions() else: readOpts) cfTable = newColFamilyTable(cfNames.mapIt($it), cfHandles) db = RocksDbReadOnlyRef( lock: createLock(), @@ -206,6 +191,7 @@ proc openRocksDbReadOnly*( path: path, dbOpts: dbOpts, readOpts: readOpts, + cfDescriptors: cfs, defaultCfHandle: cfTable.get(DEFAULT_COLUMN_FAMILY_NAME), cfTable: cfTable, ) @@ -414,15 +400,26 @@ proc close*(db: RocksDbRef) = withLock(db.lock): if not db.isClosed(): - db.dbOpts.close() - db.readOpts.close() + # the column families should be closed before the database db.cfTable.close() + rocksdb_close(db.cPtr) + db.cPtr = nil + + # opts should be closed after the database is closed + if db.dbOpts.autoClose: + db.dbOpts.close() + if db.readOpts.autoClose: + db.readOpts.close() + + for cfDesc in db.cfDescriptors: + if cfDesc.autoClose: + cfDesc.close() + if db of RocksDbReadWriteRef: let db = RocksDbReadWriteRef(db) - db.writeOpts.close() + if db.writeOpts.autoClose: + db.writeOpts.close() + rocksdb_ingestexternalfileoptions_destroy(db.ingestOptsPtr) db.ingestOptsPtr = nil - - rocksdb_close(db.cPtr) - db.cPtr = nil diff --git a/rocksdb/rocksiterator.nim b/rocksdb/rocksiterator.nim index c421981..9f0371b 100644 --- a/rocksdb/rocksiterator.nim +++ b/rocksdb/rocksiterator.nim @@ -46,8 +46,7 @@ proc seekToKey*(iter: RocksIteratorRef, key: openArray[byte]) = ## invalid. ## doAssert not iter.isClosed() - let (cKey, cLen) = (cast[cstring](unsafeAddr key[0]), csize_t(key.len)) - rocksdb_iter_seek(iter.cPtr, cKey, cLen) + rocksdb_iter_seek(iter.cPtr, cast[cstring](unsafeAddr key[0]), csize_t(key.len)) proc seekToFirst*(iter: RocksIteratorRef) = ## Seeks to the first entry in the column family. diff --git a/rocksdb/sstfilewriter.nim b/rocksdb/sstfilewriter.nim index 63b4b9b..4160850 100644 --- a/rocksdb/sstfilewriter.nim +++ b/rocksdb/sstfilewriter.nim @@ -25,10 +25,12 @@ type dbOpts: DbOptionsRef proc openSstFileWriter*( - filePath: string, dbOpts = DbOptionsRef(nil) + filePath: string, dbOpts = defaultDbOptions(autoClose = true) ): RocksDBResult[SstFileWriterRef] = ## Creates a new `SstFileWriterRef` and opens the file at the given `filePath`. - let dbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts) + ## If `dbOpts` is not supplied then the default options will be used. + ## These default options will be closed when the file writer is closed. + ## If any options are provided, they will need to be closed manually. doAssert not dbOpts.isClosed() let envOptsPtr = rocksdb_envoptions_create() @@ -42,7 +44,7 @@ proc openSstFileWriter*( rocksdb_sstfilewriter_open( writer.cPtr, filePath.cstring, cast[cstringArray](errors.addr) ) - bailOnErrors(errors) + bailOnErrors(errors, dbOpts) ok(writer) @@ -99,3 +101,6 @@ proc close*(writer: SstFileWriterRef) = writer.envOptsPtr = nil rocksdb_sstfilewriter_destroy(writer.cPtr) writer.cPtr = nil + + if writer.dbOpts.autoClose: + writer.dbOpts.close() diff --git a/rocksdb/transactiondb.nim b/rocksdb/transactiondb.nim index 71f4dfc..f449781 100644 --- a/rocksdb/transactiondb.nim +++ b/rocksdb/transactiondb.nim @@ -36,26 +36,25 @@ type path: string dbOpts: DbOptionsRef txDbOpts: TransactionDbOptionsRef + cfDescriptors: seq[ColFamilyDescriptor] defaultCfHandle: ColFamilyHandleRef cfTable: ColFamilyTableRef proc openTransactionDb*( path: string, - dbOpts = DbOptionsRef(nil), - txDbOpts = TransactionDbOptionsRef(nil), + dbOpts = defaultDbOptions(autoClose = true), + txDbOpts = defaultTransactionDbOptions(autoClose = true), columnFamilies: openArray[ColFamilyDescriptor] = [], ): RocksDBResult[TransactionDbRef] = ## Open a `TransactionDbRef` with the given options and column families. ## If no column families are provided the default column family will be used. ## If no options are provided the default options will be used. - let useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts) - defer: - if dbOpts.isNil: - useDbOpts.close() + ## These default options will be closed when the database is closed. + ## If any options are provided, they will need to be closed manually. var cfs = columnFamilies.toSeq() if DEFAULT_COLUMN_FAMILY_NAME notin columnFamilies.mapIt(it.name()): - cfs.add(defaultColFamilyDescriptor()) + cfs.add(defaultColFamilyDescriptor(autoClose = true)) var cfNames = cfs.mapIt(it.name().cstring) @@ -64,7 +63,7 @@ proc openTransactionDb*( errors: cstring let txDbPtr = rocksdb_transactiondb_open_column_families( - useDbOpts.cPtr, + dbOpts.cPtr, txDbOpts.cPtr, path.cstring, cfNames.len().cint, @@ -73,13 +72,9 @@ proc openTransactionDb*( cfHandles[0].addr, cast[cstringArray](errors.addr), ) - bailOnErrors(errors) + bailOnErrors(errors, dbOpts, txDbOpts = txDbOpts, cfDescriptors = cfs) let - dbOpts = useDbOpts # don't close on exit - txDbOpts = (if txDbOpts.isNil: defaultTransactionDbOptions() - else: txDbOpts - ) cfTable = newColFamilyTable(cfNames.mapIt($it), cfHandles) db = TransactionDbRef( lock: createLock(), @@ -87,6 +82,7 @@ proc openTransactionDb*( path: path, dbOpts: dbOpts, txDbOpts: txDbOpts, + cfDescriptors: cfs, defaultCfHandle: cfTable.get(DEFAULT_COLUMN_FAMILY_NAME), cfTable: cfTable, ) @@ -107,22 +103,17 @@ proc isClosed*(db: TransactionDbRef): bool {.inline.} = proc beginTransaction*( db: TransactionDbRef, - readOpts = ReadOptionsRef(nil), - writeOpts = WriteOptionsRef(nil), - txDbOpts = TransactionDbOptionsRef(nil), - txOpts = defaultTransactionOptions(), + readOpts = defaultReadOptions(autoClose = true), + writeOpts = defaultWriteOptions(autoClose = true), + txOpts = defaultTransactionOptions(autoClose = true), cfHandle = db.defaultCfHandle, ): TransactionRef = ## Begin a new transaction against the database. The transaction will default ## to using the specified column family. If no column family is specified ## then the default column family will be used. + ## + ## doAssert not db.isClosed() - let - txDbOpts = (if txDbOpts.isNil: defaultTransactionDbOptions() - else: txDbOpts - ) - readOpts = (if readOpts.isNil: defaultReadOptions() else: readOpts) - writeOpts = (if writeOpts.isNil: defaultWriteOptions() else: writeOpts) let txPtr = rocksdb_transaction_begin(db.cPtr, writeOpts.cPtr, txOpts.cPtr, nil) @@ -130,11 +121,21 @@ proc beginTransaction*( proc close*(db: TransactionDbRef) = ## Close the `TransactionDbRef`. + withLock(db.lock): if not db.isClosed(): - db.dbOpts.close() - db.txDbOpts.close() + # the column families should be closed before the database db.cfTable.close() rocksdb_transactiondb_close(db.cPtr) db.cPtr = nil + + # opts should be closed after the database is closed + if db.dbOpts.autoClose: + db.dbOpts.close() + if db.txDbOpts.autoClose: + db.txDbOpts.close() + + for cfDesc in db.cfDescriptors: + if cfDesc.autoClose: + cfDesc.close() diff --git a/rocksdb/transactions/transaction.nim b/rocksdb/transactions/transaction.nim index dd86c27..f268c9a 100644 --- a/rocksdb/transactions/transaction.nim +++ b/rocksdb/transactions/transaction.nim @@ -175,9 +175,13 @@ proc rollback*(tx: TransactionRef): RocksDBResult[void] = proc close*(tx: TransactionRef) = ## Close the `TransactionRef`. if not tx.isClosed(): - tx.readOpts.close() - tx.writeOpts.close() - tx.txOpts.close() - rocksdb_transaction_destroy(tx.cPtr) tx.cPtr = nil + + # opts should be closed after the transaction is closed + if tx.readOpts.autoClose: + tx.readOpts.close() + if tx.writeOpts.autoClose: + tx.writeOpts.close() + if tx.txOpts.autoClose: + tx.txOpts.close() diff --git a/rocksdb/transactions/txdbopts.nim b/rocksdb/transactions/txdbopts.nim index 6cb0fea..db019cb 100644 --- a/rocksdb/transactions/txdbopts.nim +++ b/rocksdb/transactions/txdbopts.nim @@ -16,9 +16,12 @@ type TransactionDbOptionsRef* = ref object cPtr: TransactionDbOptionsPtr + autoClose*: bool # if true then close will be called when the database is closed -proc newTransactionDbOptions*(): TransactionDbOptionsRef = - TransactionDbOptionsRef(cPtr: rocksdb_transactiondb_options_create()) +proc newTransactionDbOptions*(autoClose = false): TransactionDbOptionsRef = + TransactionDbOptionsRef( + cPtr: rocksdb_transactiondb_options_create(), autoClose: autoClose + ) proc isClosed*(txDbOpts: TransactionDbOptionsRef): bool {.inline.} = txDbOpts.cPtr.isNil() @@ -29,8 +32,10 @@ proc cPtr*(txDbOpts: TransactionDbOptionsRef): TransactionDbOptionsPtr = # TODO: Add setters and getters for backup options properties. -proc defaultTransactionDbOptions*(): TransactionDbOptionsRef {.inline.} = - newTransactionDbOptions() +proc defaultTransactionDbOptions*( + autoClose = false +): TransactionDbOptionsRef {.inline.} = + newTransactionDbOptions(autoClose) # TODO: set prefered defaults proc close*(txDbOpts: TransactionDbOptionsRef) = diff --git a/rocksdb/transactions/txopts.nim b/rocksdb/transactions/txopts.nim index 395be16..e31d7c7 100644 --- a/rocksdb/transactions/txopts.nim +++ b/rocksdb/transactions/txopts.nim @@ -16,9 +16,12 @@ type TransactionOptionsRef* = ref object cPtr: TransactionOptionsPtr + autoClose*: bool # if true then close will be called when the transaction is closed -proc newTransactionOptions*(): TransactionOptionsRef = - TransactionOptionsRef(cPtr: rocksdb_transaction_options_create()) +proc newTransactionOptions*(autoClose = false): TransactionOptionsRef = + TransactionOptionsRef( + cPtr: rocksdb_transaction_options_create(), autoClose: autoClose + ) proc isClosed*(txOpts: TransactionOptionsRef): bool {.inline.} = txOpts.cPtr.isNil() @@ -29,8 +32,8 @@ proc cPtr*(txOpts: TransactionOptionsRef): TransactionOptionsPtr = # TODO: Add setters and getters for backup options properties. -proc defaultTransactionOptions*(): TransactionOptionsRef {.inline.} = - newTransactionOptions() +proc defaultTransactionOptions*(autoClose = false): TransactionOptionsRef {.inline.} = + newTransactionOptions(autoClose) # TODO: set prefered defaults proc close*(txOpts: TransactionOptionsRef) = diff --git a/tests/columnfamily/test_cfdescriptor.nim b/tests/columnfamily/test_cfdescriptor.nim index 7951656..3fa3a89 100644 --- a/tests/columnfamily/test_cfdescriptor.nim +++ b/tests/columnfamily/test_cfdescriptor.nim @@ -9,21 +9,11 @@ {.used.} -import unittest2, ../../rocksdb/internal/utils, ../../rocksdb/columnfamily/cfdescriptor +import unittest2, ../../rocksdb/columnfamily/cfdescriptor suite "ColFamilyDescriptor Tests": const TEST_CF_NAME = "test" - test "Test initColFamilyDescriptor": - var descriptor = initColFamilyDescriptor(TEST_CF_NAME) - - check: - descriptor.name() == TEST_CF_NAME - not descriptor.options().isNil() - not descriptor.isDefault() - - descriptor.close() - test "Test initColFamilyDescriptor with options": var descriptor = initColFamilyDescriptor(TEST_CF_NAME, defaultColFamilyOptions()) diff --git a/tests/test_helper.nim b/tests/test_helper.nim index b0eae61..37d2e1e 100644 --- a/tests/test_helper.nim +++ b/tests/test_helper.nim @@ -15,7 +15,10 @@ proc initReadWriteDb*( path: string, columnFamilyNames: openArray[string] = @[] ): RocksDbReadWriteRef = let res = openRocksDb( - path, columnFamilies = columnFamilyNames.mapIt(initColFamilyDescriptor(it)) + path, + columnFamilies = columnFamilyNames.mapIt( + initColFamilyDescriptor(it, defaultColFamilyOptions(autoClose = true)) + ), ) if res.isErr(): echo res.error() @@ -26,7 +29,10 @@ proc initReadOnlyDb*( path: string, columnFamilyNames: openArray[string] = @[] ): RocksDbReadOnlyRef = let res = openRocksDbReadOnly( - path, columnFamilies = columnFamilyNames.mapIt(initColFamilyDescriptor(it)) + path, + columnFamilies = columnFamilyNames.mapIt( + initColFamilyDescriptor(it, defaultColFamilyOptions(autoClose = true)) + ), ) if res.isErr(): echo res.error() @@ -43,8 +49,9 @@ proc initTransactionDb*( ): TransactionDbRef = let res = openTransactionDb( path, - txDbOpts = defaultTransactionDbOptions(), - columnFamilies = columnFamilyNames.mapIt(initColFamilyDescriptor(it)), + columnFamilies = columnFamilyNames.mapIt( + initColFamilyDescriptor(it, defaultColFamilyOptions(autoClose = true)) + ), ) if res.isErr(): echo res.error() diff --git a/tests/test_rocksdb.nim b/tests/test_rocksdb.nim index f6070a0..7c3480d 100644 --- a/tests/test_rocksdb.nim +++ b/tests/test_rocksdb.nim @@ -231,18 +231,6 @@ suite "RocksDbRef Tests": readOnlyDb.close() check readOnlyDb.isClosed() - test "Close multiple times": - check not db.isClosed() - db.close() - check db.isClosed() - db.close() - check db.isClosed() - - test "Unknown column family": - const CF_UNKNOWN = "unknown" - let cfHandleRes = db.getColFamilyHandle(CF_UNKNOWN) - check cfHandleRes.isErr() and cfHandleRes.error() == "rocksdb: unknown column family" - test "Test missing key and values": let key1 = @[byte(1)] # exists with non empty value @@ -335,3 +323,48 @@ suite "RocksDbRef Tests": r.value() == false v.len() == 0 db.get(key5).isErr() + + test "List column familes": + let cfRes1 = listColumnFamilies(dbPath) + check: + cfRes1.isOk() + cfRes1.value() == @[CF_DEFAULT, CF_OTHER] + + let + dbPath2 = dbPath & "2" + db2 = initReadWriteDb(dbPath2, columnFamilyNames = @[CF_DEFAULT]) + cfRes2 = listColumnFamilies(dbPath2) + check: + cfRes2.isOk() + cfRes2.value() == @[CF_DEFAULT] + + test "Unknown column family": + const CF_UNKNOWN = "unknown" + let cfHandleRes = db.getColFamilyHandle(CF_UNKNOWN) + check cfHandleRes.isErr() and cfHandleRes.error() == "rocksdb: unknown column family" + + test "Close multiple times": + check not db.isClosed() + db.close() + check db.isClosed() + db.close() + check db.isClosed() + + test "Test auto close": + let + dbPath = mkdtemp() / "autoclose" + dbOpts = defaultDbOptions(autoClose = false) + readOpts = defaultReadOptions(autoClose = true) + db = openRocksDb(dbPath, dbOpts, readOpts).get() + + check: + dbOpts.isClosed() == false + readOpts.isClosed() == false + db.isClosed() == false + + db.close() + + check: + dbOpts.isClosed() == false + readOpts.isClosed() == true + db.isClosed() == true diff --git a/tests/test_rocksiterator.nim b/tests/test_rocksiterator.nim index 348ed96..b8c2fae 100644 --- a/tests/test_rocksiterator.nim +++ b/tests/test_rocksiterator.nim @@ -151,6 +151,20 @@ suite "RocksIteratorRef Tests": iter2.key() == @[byte(3)] iter2.value() == @[byte(3)] + test "Iterate forwards using seek to key": + let res = db.openIterator(defaultCfHandle) + check res.isOk() + + var iter = res.get() + defer: + iter.close() + + iter.seekToKey(key2) + check: + iter.isValid() + iter.key() == key2 + iter.value() == val2 + test "Empty column family": let res = db.openIterator(emptyCfHandle) check res.isOk() diff --git a/tests/test_transactiondb.nim b/tests/test_transactiondb.nim index bddde84..c991c73 100644 --- a/tests/test_transactiondb.nim +++ b/tests/test_transactiondb.nim @@ -201,3 +201,22 @@ suite "TransactionDbRef Tests": check tx2.isClosed() tx2.close() check tx2.isClosed() + + test "Test auto close": + let + dbPath = mkdtemp() / "autoclose" + dbOpts = defaultDbOptions(autoClose = false) + txDbOpts = defaultTransactionDbOptions(autoClose = true) + db = openTransactionDb(dbPath, dbOpts, txDbOpts).get() + + check: + dbOpts.isClosed() == false + txDbOpts.isClosed() == false + db.isClosed() == false + + db.close() + + check: + dbOpts.isClosed() == false + txDbOpts.isClosed() == true + db.isClosed() == true diff --git a/tests/test_writebatch.nim b/tests/test_writebatch.nim index cc1ec6c..ac219e5 100644 --- a/tests/test_writebatch.nim +++ b/tests/test_writebatch.nim @@ -65,18 +65,18 @@ suite "WriteBatchRef Tests": not batch.isClosed() test "Test writing batch to column family": - var batch = db.openWriteBatch(otherCfHandle) + var batch = db.openWriteBatch() defer: batch.close() check not batch.isClosed() check: - batch.put(key1, val1).isOk() - batch.put(key2, val2).isOk() - batch.put(key3, val3).isOk() + batch.put(key1, val1, otherCfHandle).isOk() + batch.put(key2, val2, otherCfHandle).isOk() + batch.put(key3, val3, otherCfHandle).isOk() batch.count() == 3 - batch.delete(key2).isOk() + batch.delete(key2, otherCfHandle).isOk() batch.count() == 4 not batch.isClosed() @@ -92,26 +92,54 @@ suite "WriteBatchRef Tests": batch.count() == 0 not batch.isClosed() + test "Test writing to multiple column families in single batch": + var batch = db.openWriteBatch() + defer: + batch.close() + check not batch.isClosed() + + check: + batch.put(key1, val1, defaultCfHandle).isOk() + batch.put(key1, val1, otherCfHandle).isOk() + batch.put(key2, val2, otherCfHandle).isOk() + batch.put(key3, val3, otherCfHandle).isOk() + batch.count() == 4 + + batch.delete(key2, otherCfHandle).isOk() + batch.count() == 5 + not batch.isClosed() + + let res = db.write(batch) + check: + res.isOk() + db.get(key1, defaultCfHandle).get() == val1 + db.get(key1, otherCfHandle).get() == val1 + db.keyExists(key2, otherCfHandle).get() == false + db.get(key3, otherCfHandle).get() == val3 + + batch.clear() + check: + batch.count() == 0 + not batch.isClosed() + test "Test writing to multiple column families in multiple batches": - var batch1 = db.openWriteBatch(defaultCfHandle) + var batch1 = db.openWriteBatch() defer: batch1.close() check not batch1.isClosed() - var batch2 = db.openWriteBatch(otherCfHandle) + var batch2 = db.openWriteBatch() defer: batch2.close() check not batch2.isClosed() check: batch1.put(key1, val1).isOk() - batch1.delete(key2).isOk() - batch1.put(key3, val3).isOk() - - batch2.put(key1, val1).isOk() - batch2.delete(key1).isOk() + batch1.delete(key2, otherCfHandle).isOk() + batch1.put(key3, val3, otherCfHandle).isOk() + batch2.put(key1, val1, otherCfHandle).isOk() + batch2.delete(key1, otherCfHandle).isOk() batch2.put(key3, val3).isOk() - batch1.count() == 3 batch2.count() == 3