diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 4d39e3b..3a44fa9 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -13,9 +13,11 @@ jobs: bun_ci: uses: cross-org/workflows/.github/workflows/bun-ci.yml@main with: - jsr_dependencies: "@cross/test @cross/fs @std/assert @std/path cbor-x" + jsr_dependencies: "@cross/test @cross/fs @std/assert @std/path" + npm_dependencies: "cbor-x" node_ci: uses: cross-org/workflows/.github/workflows/node-ci.yml@main with: jsr_dependencies: "@cross/test @cross/fs @std/assert @std/path cbor-x" + npm_dependencies: "cbor-x" test_target: "src/*.test.ts" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 6e9a237..cb3d905 100644 --- a/.gitignore +++ b/.gitignore @@ -16,127 +16,19 @@ pids *.seed *.pid.lock -# Directory for instrumented libs generated by jscoverage/JSCover -lib-cov - # Coverage directory used by tools like istanbul coverage *.lcov -# nyc test coverage -.nyc_output - -# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) -.grunt - -# Bower dependency directory (https://bower.io/) -bower_components - -# node-waf configuration -.lock-wscript - -# Compiled binary addons (https://nodejs.org/api/addons.html) -build/Release - -# Dependency directories +# Node / NPM node_modules/ -jspm_packages/ - -# Snowpack dependency directory (https://snowpack.dev/) -web_modules/ - -# TypeScript cache -*.tsbuildinfo - -# Optional npm cache directory -.npm - -# Optional eslint cache -.eslintcache - -# Optional stylelint cache -.stylelintcache - -# Microbundle cache -.rpt2_cache/ -.rts2_cache_cjs/ -.rts2_cache_es/ -.rts2_cache_umd/ - -# Optional REPL history -.node_repl_history - -# Output of 'npm pack' -*.tgz - -# Yarn Integrity file -.yarn-integrity - -# dotenv environment variable files -.env -.env.development.local -.env.test.local -.env.production.local -.env.local - -# parcel-bundler cache (https://parceljs.org/) -.cache -.parcel-cache - -# Next.js build output -.next -out - -# Nuxt.js build / generate output -.nuxt -dist - -# Gatsby files -.cache/ -# Comment in the public line in if your project uses Gatsby and not Next.js -# https://nextjs.org/blog/next-9-1#public-directory-support -# public - -# vuepress build output -.vuepress/dist - -# vuepress v2.x temp and cache directory -.temp -.cache - -# Docusaurus cache and generated files -.docusaurus - -# Serverless directories -.serverless/ - -# FuseBox cache -.fusebox/ - -# DynamoDB Local files -.dynamodb/ - -# TernJS port file -.tern-port - -# Stores VSCode versions used for testing VSCode extensions -.vscode-test - -# yarn v2 -.yarn/cache -.yarn/unplugged -.yarn/build-state.yml -.yarn/install-state.gz -.pnp.* - -# Deno / JSR -deno.lock - -# NPM .npmrc package-lock.json package.json +# Deno / JSR +deno.lock + # Bun bunfig.toml diff --git a/README.md b/README.md index cc0a92d..297c3d7 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ **cross/kv** A cross-platform, hierarchical Key/Value database for JavaScript and TypeScript, -that work in all major runtimes (Node.js, Deno and Bun). +designed to work in all major runtimes (Node.js, Deno, and Bun). ### **Features** @@ -21,13 +21,14 @@ Full installation instructions available at ```bash # Using npm -npx jsr i @cross/kv +npx jsr add @cross/kv # Using Deno deno add @cross/kv -``` -Replace `` with the desired version of the package. +# Using bun +bunx jsr add @cross/kv +``` ### **Simple Usage** @@ -42,7 +43,7 @@ await kvStore.set(["data", "username"], "Alice"); // Get a value const username = await kvStore.get(["data", "username"]); -console.log(username); // Output: 'Alice' +console.log(username); // Output: { ts: , data "Alice" } // Delete a key await kvStore.delete(["data", "username"]); @@ -110,11 +111,12 @@ await kvStore.close(); - `CrossKV` class - `open(filepath)` - - `set(key, value)` + - `set(key, value, overwrite?)` - `get(key)` - `getMany(key)` - `delete(key)` - - `sync()` + - `beginTransaction()` + - `endTransaction()` - `close()` - `KVKey` class (Detail the constructor and methods) - `KVKeyRange` interface diff --git a/deno.json b/deno.json index be93c2f..353acf7 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "name": "@cross/kv", - "version": "0.0.4", + "version": "0.0.5", "exports": { ".": "./mod.ts" }, @@ -13,7 +13,7 @@ "cbor-x": "npm:cbor-x@^1.5.9" }, "publish": { - "exclude": [".github", "*.test.ts", "*.bench.ts"] + "exclude": [".github", "**/*.test.ts", "**/*.bench.ts"] }, "tasks": { "check": "deno fmt --check && deno lint && deno check mod.ts && deno doc --lint mod.ts && deno run -A mod.ts --slim --ignore-unused", diff --git a/mod.ts b/mod.ts index 4f75b75..e002665 100644 --- a/mod.ts +++ b/mod.ts @@ -1,3 +1,2 @@ export * from "./src/kv.ts"; export * from "./src/key.ts"; -export * from "./src/index.ts"; diff --git a/src/cbor.ts b/src/cbor.ts new file mode 100644 index 0000000..8820855 --- /dev/null +++ b/src/cbor.ts @@ -0,0 +1,21 @@ +import { addExtension, Decoder, Encoder } from "cbor-x"; +import { KVKey } from "./key.ts"; + +addExtension({ + Class: KVKey, + tag: 43331, // register our own extension code (a tag code) + //@ts-ignore external + encode(instance, encode) { + // define how your custom class should be encoded + // @ts-ignore external + encode(instance.get()); // return a buffer + }, + //@ts-ignore external + decode(data) { + // @ts-ignore external + return new KVKey(data as (string | number)[]); // decoded value from buffer + }, +}); + +export const extEncoder = new Encoder(); +export const extDecoder = new Decoder(); diff --git a/src/index.ts b/src/index.ts index 2217348..2cb68bc 100644 --- a/src/index.ts +++ b/src/index.ts @@ -116,7 +116,10 @@ export class KVIndex { if (childNode) { recurse(childNode, keyIndex + 1); } - } else if (typeof keyPart === "object" && keyPart.from && keyPart.to) { + } else if ( + typeof keyPart === "object" && + (keyPart.from !== undefined || keyPart.to !== undefined) + ) { // Key range const range = keyPart as KVKeyRange; for (const [index, childNode] of node.children.entries()) { diff --git a/src/key.ts b/src/key.ts index 7ad297f..309e8f5 100644 --- a/src/key.ts +++ b/src/key.ts @@ -39,9 +39,9 @@ export class KVKey { } if (typeof element === "object") { - if (!element.from || !element.to) { + if (!(element.from || element.to)) { throw new TypeError( - 'Ranges must have both "from" and "to" properties', + 'Ranges must have one or both of "from" and "to"', ); } } diff --git a/src/kv.bench.ts b/src/kv.bench.ts index be92cea..8a8c90d 100644 --- a/src/kv.bench.ts +++ b/src/kv.bench.ts @@ -27,19 +27,25 @@ let crossIter = 0; let denoIter = 0; await Deno.bench("cross_kv_set", async () => { - await crossStore.set(["testKey", crossIter++], { data: "testData" }); + await crossStore.set(["testKey", crossIter++], { + data: "testData", + ts: new Date(), + }); }); await Deno.bench("deno_kv_set", async () => { - await denoStore.set(["testKey", denoIter++], { data: "testData" }); + await denoStore.set(["testKey", denoIter++], { + data: "testData", + ts: new Date(), + }); }); await Deno.bench("cross_kv_get", async () => { - await crossStore.get(["testKey", crossIter--]); + await crossStore.get(["testKey", 3]); }); await Deno.bench("deno_kv_get", async () => { - await denoStore.get(["testKey", crossIter--]); + await denoStore.get(["testKey", 3]); }); //await cleanup(); diff --git a/src/kv.test.ts b/src/kv.test.ts index 9728e2d..b528d40 100644 --- a/src/kv.test.ts +++ b/src/kv.test.ts @@ -1,4 +1,4 @@ -import { assertEquals, assertNotEquals, assertRejects } from "@std/assert"; +import { assertEquals, assertRejects } from "@std/assert"; import { test } from "@cross/test"; import { CrossKV } from "./kv.ts"; import { tempfile } from "@cross/fs"; @@ -10,13 +10,13 @@ test("CrossKV: set, get and delete (numbers and strings)", async () => { await kvStore.set(["name"], "Alice"); await kvStore.set(["age"], 30); - assertEquals(await kvStore.get(["name"]), "Alice"); - assertEquals(await kvStore.get(["age"]), 30); + assertEquals((await kvStore.get(["name"]))?.data, "Alice"); + assertEquals((await kvStore.get(["age"]))?.data, 30); await kvStore.delete(["name"]); assertEquals(await kvStore.get(["name"]), null); - assertEquals(await kvStore.get(["age"]), 30); + assertEquals((await kvStore.get(["age"]))?.data, 30); }); test("CrossKV: set, get and delete (objects)", async () => { @@ -26,13 +26,13 @@ test("CrossKV: set, get and delete (objects)", async () => { await kvStore.set(["name"], { data: "Alice" }); await kvStore.set(["age"], { data: 30 }); - assertEquals(await kvStore.get(["name"]), { data: "Alice" }); - assertEquals(await kvStore.get(["age"]), { data: 30 }); + assertEquals((await kvStore.get(["name"]))?.data, { data: "Alice" }); + assertEquals((await kvStore.get(["age"]))?.data, { data: 30 }); await kvStore.delete(["name"]); assertEquals(await kvStore.get(["name"]), null); - assertEquals(await kvStore.get(["age"]), { data: 30 }); + assertEquals((await kvStore.get(["age"]))?.data, { data: 30 }); }); test("CrossKV: set, get and delete (dates)", async () => { @@ -41,9 +41,12 @@ test("CrossKV: set, get and delete (dates)", async () => { await kvStore.open(tempFilePrefix); const date = new Date(); await kvStore.set(["pointintime"], date); - assertEquals((await kvStore.get(["pointintime"])).getTime(), date.getTime()); assertEquals( - (await kvStore.get(["pointintime"])).toLocaleString(), + ((await kvStore.get(["pointintime"]))!.data! as Date).getTime(), + date.getTime(), + ); + assertEquals( + (await kvStore.get(["pointintime"]))?.data?.toLocaleString(), date.toLocaleString(), ); await kvStore.delete(["pointintime"]); @@ -83,8 +86,8 @@ test("CrossKV: supports multi-level nested keys", async () => { await kvStore.set(["data", "user", "name"], "Alice"); await kvStore.set(["data", "system", "version"], 1.2); - assertEquals(await kvStore.get(["data", "user", "name"]), "Alice"); - assertEquals(await kvStore.get(["data", "system", "version"]), 1.2); + assertEquals((await kvStore.get(["data", "user", "name"]))?.data, "Alice"); + assertEquals((await kvStore.get(["data", "system", "version"]))?.data, 1.2); }); test("CrossKV: supports multi-level nested keys with numbers", async () => { @@ -95,9 +98,9 @@ test("CrossKV: supports multi-level nested keys with numbers", async () => { await kvStore.set(["data", "user", 4], "Alice"); await kvStore.set(["data", "system", 4], 1.2); - assertEquals(await kvStore.get(["data", "user", 4]), "Alice"); - assertEquals(await kvStore.get(["data", "system", 4]), 1.2); - assertNotEquals(await kvStore.get(["data", "system", 5]), 1.2); + assertEquals((await kvStore.get(["data", "user", 4]))?.data, "Alice"); + assertEquals((await kvStore.get(["data", "system", 4]))?.data, 1.2); + assertEquals(await kvStore.get(["data", "system", 5]), null); }); test("CrossKV: supports numeric key ranges", async () => { @@ -112,11 +115,10 @@ test("CrossKV: supports numeric key ranges", async () => { // Test if the 'get' function returns the expected values const rangeResults = await kvStore.getMany(["data", { from: 7, to: 9 }]); - assertEquals(rangeResults, [ - `Value 7`, - `Value 8`, - `Value 9`, - ]); + assertEquals(rangeResults.length, 3); + assertEquals(rangeResults[0].data, "Value 7"); + assertEquals(rangeResults[1].data, "Value 8"); + assertEquals(rangeResults[2].data, "Value 9"); }); test("CrossKV: supports string key ranges", async () => { @@ -134,8 +136,6 @@ test("CrossKV: supports string key ranges", async () => { from: "doc_", to: "doc_z", }]); - assertEquals(rangeResults, [ - "Document A", - "Document B", - ]); + assertEquals(rangeResults.length, 2); + assertEquals(rangeResults[0].data, "Document A"); }); diff --git a/src/kv.ts b/src/kv.ts index ee270b8..63d9bb1 100644 --- a/src/kv.ts +++ b/src/kv.ts @@ -8,27 +8,22 @@ import { KVOperation, type KVPendingTransaction, } from "./transaction.ts"; +import { extDecoder, extEncoder } from "./cbor.ts"; -import { addExtension, Decoder, Encoder } from "cbor-x"; - -const extEncoder = new Encoder(); -const extDecoder = new Decoder(); - -addExtension({ - Class: KVKey, - tag: 43331, // register our own extension code (a tag code) - //@ts-ignore external - encode(instance, encode) { - // define how your custom class should be encoded - // @ts-ignore external - encode(instance.get()); // return a buffer - }, - //@ts-ignore external - decode(data) { - // @ts-ignore external - return new KVKey(data as (string | number)[]); // decoded value from buffer - }, -}); +/** + * Represents a single data entry within the Key-Value store. + */ +export interface KVDataEntry { + /** + * The timestamp (milliseconds since epoch) when the entry was created or modified. + */ + ts: number; + + /** + * The actual data stored in the Key-Value store. Can be any type. + */ + data: unknown; +} /** * Cross-platform Key-Value store implementation backed by file storage. @@ -42,12 +37,12 @@ export class CrossKV { private dataPath?: string; private transactionsPath?: string; - constructor() { - } + constructor() {} /** * Opens the Key-Value store based on a provided file path. * Initializes the index and data files. + * * @param filePath - Path to the base file for the KV store. * Index and data files will be derived from this path. */ @@ -63,11 +58,21 @@ export class CrossKV { await this.restoreTransactionLog(); } + /** + * Begins a new transaction. + * @throws {Error} If already in a transaction. + */ public beginTransaction() { if (this.isInTransaction) throw new Error("Already in a transaction"); this.isInTransaction = true; } + /** + * Ends the current transaction, executing all pending operations. + * + * @returns {Promise} A promise resolving to an array of errors + * encountered during transaction execution (empty if successful). + */ public async endTransaction(): Promise { if (!this.isInTransaction) throw new Error("Not in a transaction"); @@ -92,6 +97,8 @@ export class CrossKV { /** * Loads all KVFinishedTransaction entries from the transaction log and * replays them to rebuild the index state. + * + * @private */ private async restoreTransactionLog() { this.ensureOpen(); @@ -118,8 +125,8 @@ export class CrossKV { this.index.delete(transaction); break; } - } catch (e) { - //console.error(e); + } catch (_e) { + throw new Error("Error while encoding data"); } position += 2 + dataLength; // Move to the next transaction @@ -128,6 +135,8 @@ export class CrossKV { /** * Ensures the database is open, throwing an error if it's not. + * + * @private * @throws {Error} If the database is not open. */ private ensureOpen(): void { @@ -137,12 +146,12 @@ export class CrossKV { } /** - * Retrieves the first value associated with the given key (limit). + * Retrieves the first value associated with the given key, or null. + * * @param key - Representation of the key. - * @param limit - Optional maximum number of values to retrieve. - * @returns the retrieved value, or null + * @returns The retrieved value, or null if not found. */ - public async get(key: KVKeyRepresentation): Promise { + public async get(key: KVKeyRepresentation): Promise { const result = await this.getMany(key, 1); if (result.length) { return result[0]; @@ -152,12 +161,16 @@ export class CrossKV { } /** - * Retrieves one or more values associated with the given key (limit). + * Retrieves all values associated with the given key, with an optional record limit. + * * @param key - Representation of the key. * @param limit - Optional maximum number of values to retrieve. * @returns An array of retrieved values. */ - async getMany(key: KVKeyRepresentation, limit?: number): Promise { + async getMany( + key: KVKeyRepresentation, + limit?: number, + ): Promise { this.ensureOpen(); const validatedKey = new KVKey(key, true); const offsets = this.index!.get(validatedKey)!; @@ -195,18 +208,34 @@ export class CrossKV { * @param encodedData - The CBOR-encoded value to write. * @returns The offset at which the data was written. */ - private async writeData(encodedData: Uint8Array): Promise { + private async writeData( + pendingTransaction: KVPendingTransaction, + ): Promise { this.ensureOpen(); + // Create a data entry + const dataEntry: KVDataEntry = { + ts: pendingTransaction.ts, + data: pendingTransaction.data, + }; + + const encodedDataEntry = extEncoder.encode(dataEntry); + // Get current offset (since we're appending) const stats = await stat(this.dataPath!); // Use fs.fstat instead const originalPosition = stats.size; + // Create array + const fullData = new Uint8Array(2 + encodedDataEntry.length); + // Add length prefix (2 bytes) - const fullData = new Uint8Array(2 + encodedData.length); - new DataView(fullData.buffer).setUint16(0, encodedData.length, false); - fullData.set(encodedData, 2); + new DataView(fullData.buffer).setUint16(0, encodedDataEntry.length, false); + + // Add data + fullData.set(encodedDataEntry, 2); + await writeAtPosition(this.dataPath!, fullData, originalPosition); + return originalPosition; // Return the offset (where the write started) } @@ -247,15 +276,12 @@ export class CrossKV { // Ensure the key is ok const validatedKey = new KVKey(key); - // Encode data - const encodedData = extEncoder.encode(value); - // Create transaction const pendingTransaction: KVPendingTransaction = { key: validatedKey, oper: overwrite ? KVOperation.UPSERT : KVOperation.INSERT, ts: new Date().getTime(), - data: encodedData, + data: value, }; // Enqueue transaction @@ -331,7 +357,7 @@ export class CrossKV { // 12. Write Data if Needed let offset; if (pendingTransaction.data !== undefined) { - offset = await this.writeData(pendingTransaction.data); + offset = await this.writeData(pendingTransaction); } // 3. Create the finished transaction diff --git a/src/transaction.ts b/src/transaction.ts index a5725ea..cd9d612 100644 --- a/src/transaction.ts +++ b/src/transaction.ts @@ -53,5 +53,5 @@ export interface KVPendingTransaction { /** * Actual data for this transaction, ready to be written */ - data?: Uint8Array; + data?: unknown; }