diff --git a/CHANGELOG.md b/CHANGELOG.md index 241c07a..0cdff8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ ## Changes - Remove sync result `noop` as isn't used anywhere anymore +- Adds `fetchData` option to `scan`. Setting this to `false` enables faster + retrieval of transaction metadata. ## Fixes diff --git a/README.md b/README.md index 6f9550f..e9e8924 100644 --- a/README.md +++ b/README.md @@ -107,31 +107,36 @@ deno install -frA --name ckv jsr:@cross/kv/cli ### Methods - `KV(options)` - Main class. Options are optional. - - `async open(filepath, createIfMissing)` - Opens the KV store. - `createIfMissing` defaults to true. - - `async set(key, value)` - Stores a value. - - `async get(key)` - Retrieves a value. - - `async *iterate(query, limit, reverse)` - Iterates over entries for a - key. Limit and reverse are optional. - - `async listAll(query, limit, reverse)` - Gets all entries for a key as an - array. Limit and reverse are optional. - - `listKeys(query)` - List all keys under . - - `async delete(key)` - Deletes a key-value pair. - - `async sync()` - Synchronizez the ledger with disk. + - `async open(filepath, createIfMissing = true)` - OOpens the KV store at the + specified file path, creating it if it doesn't exist (default behavior). + - `async set(key, value)` - Stores a value associated with the given key. + - `async delete(key)` - Removes the key-value pair identified by the key. + - `async get(key)` - Retrieves the value associated with the specified key. + Returns null if the key does not exist. + - `async *iterate(query, limit, reverse)` - Asynchronously iterates over + the latest values matching the query + - `async listAll(query, limit, reverse)` - Retrieves all latest values + matching the query as an array. + - `async *scan(query, limit, reverse)` - Asynchronously iterates over the + transaction history (all set and delete operations) for keys matching the + query. Optionally recurses into subkeys and fetches the associated data. + - `listKeys(query)` - Returns an array of all keys matching the given query. + - `async sync()` - Manually synchronizes the in-memory index with the on-disk + data store. - `watch(query, callback, recursive): void` - Registers a callback to be - called whenever a new transaction matching the given query is added to the - database. + invoked whenever a matching transaction (set or delete) is added. - `unwatch(query, callback): void` - Unregisters a previously registered watch handler. - - `beginTransaction()` - Starts an atomic transaction. - - `async endTransaction()` - Ends an atomic transaction, returns a list of - `Errors` if any occurred. - - `async vacuum()` - Reclaims storage space. - - `on(eventName, eventData)` - Listen for events such as `sync`, - `watchdogError` or `closing`. - - `isOpen()` - Returns true if the database is open and ready for - transactions. - - `async close()` - Closes the KV store. + - `beginTransaction()` - Starts an atomic transaction, ensuring data + consistency for multiple operations. + - `async endTransaction()` - Commits all changes made within the transaction, + or rolls back if errors occur. + - `async vacuum()` - Optimizes storage by removing redundant transaction + history, retaining only the latest value for each key. + - `on(eventName, eventData)` - Subscribes to events like `sync`, + `watchdogError`, or `closing` to get notified of specific occurrences. + - `isOpen()` - Returns true if the database is open and ready for operations. + - `async close()` - Closes the KV store, ensuring resources are released. ### Keys diff --git a/src/cli/commands/stats.ts b/src/cli/commands/stats.ts index 0f10e28..c6d8264 100644 --- a/src/cli/commands/stats.ts +++ b/src/cli/commands/stats.ts @@ -71,7 +71,7 @@ async function stats( let ledgerInvalidCount = 0; if (kvStore) { try { - for await (const entry of kvStore.scan([], true)) { + for await (const entry of kvStore.scan([], true, false)) { if (entry.operation === KVOperation.SET) { ledgerSetCount++; } else if (entry.operation === KVOperation.DELETE) { diff --git a/src/lib/kv.ts b/src/lib/kv.ts index d4078be..4d7ff90 100644 --- a/src/lib/kv.ts +++ b/src/lib/kv.ts @@ -345,18 +345,20 @@ export class KV extends EventEmitter { } /** - * Asynchronously iterates over data entries associated with a given key. + * Asynchronously iterates over transactions associated with a given key. * * @param query - Representation of the key to search for, or a query object for complex filters. * @param recursive - Match all entries matching the given query, and recurse. + * @param fetchData - Return transactions with full data. Setting this to false improves performance, but does only yield transaction metadata. * @returns An async generator yielding `KVTransactionResult` objects for each matching entry. */ public async *scan( query: KVKey | KVQuery, recursive: boolean = false, + fetchData: boolean = true, ): AsyncGenerator> { this.ensureOpen(); - for await (const result of this.ledger!.scan(query, recursive)) { + for await (const result of this.ledger!.scan(query, recursive, fetchData)) { if (result?.transaction) { // Null check to ensure safety const processedResult = result.transaction.asResult(); // Apply your processing logic here yield processedResult; diff --git a/src/lib/ledger.ts b/src/lib/ledger.ts index c62026a..a9a1017 100644 --- a/src/lib/ledger.ts +++ b/src/lib/ledger.ts @@ -120,7 +120,7 @@ export class KVLedger { try { await this.readHeader(); - // If the ledger is re-created (by vacuum or overwriting), there will be a time in the cached header + // If the ledger is re-created (by vacuum or overwriting), there will be one time in the cached header // and there will be a different time after reading the header if (currentCreated !== 0 && currentCreated !== this.header.created) { // Return 0 to invalidate this ledger @@ -205,11 +205,13 @@ export class KVLedger { * * @param query * @param recursive + * @param fetchData * @returns An async generator yielding `KVLedgerResult` objects for each matching transaction. */ public async *scan( query: KVQuery, recursive: boolean, + fetchData: boolean = true, ): AsyncIterableIterator { this.ensureOpen(); @@ -228,7 +230,7 @@ export class KVLedger { ); if (result.transaction.key?.matchesQuery(query, recursive)) { // Check for completeness - if (result.complete) { + if (result.complete || !fetchData) { yield result; } else { const completeResult = await this.rawGetTransaction( @@ -335,7 +337,7 @@ export class KVLedger { try { if (!externalFd) fd = await rawOpen(this.dataPath, false); - // Fetch 2 + 4 + 4 bytes (signature, header length, data length) + prefetch + // Fetch 2 + 4 + 4 bytes (signature, header length, data length) const transactionLengthData = await readAtPosition( fd!, TRANSACTION_SIGNATURE.length + 4 + 4, diff --git a/src/lib/transaction.ts b/src/lib/transaction.ts index ca852de..27e3366 100644 --- a/src/lib/transaction.ts +++ b/src/lib/transaction.ts @@ -139,10 +139,10 @@ export class KVTransaction { } public headerFromUint8Array( - data: Uint8Array | DataView, + data: Uint8Array, readHash: boolean = true, ) { - const dataView = data instanceof DataView ? data : new DataView( + const dataView = new DataView( data.buffer, data.byteOffset, data.byteLength, @@ -167,11 +167,9 @@ export class KVTransaction { // Decode hash bytes if (readHash) { - this.hash = new Uint8Array( - dataView.buffer.slice( - dataView.byteOffset + offset, - dataView.byteOffset + offset + hashLength, - ), + this.hash = data.slice( + offset, + offset + hashLength, ); } offset += hashLength;