feat(kv): introduce KVBufferStore interface and implement filesystem …

…based KVBufferStore

We will introduce new implementations of the KVBufferStore (e.g. LMDB) and use it new implementations of existing interfaces for transaction and header data. The filesystem based implementatin extends existing functionality and is useful for testing.

src/store/fs-kv-store.ts

@@ -0,0 +1,104 @@
+/**
+ * AR.IO Gateway
+ * Copyright (C) 2022-2023 Permanent Data Solutions, Inc. All Rights Reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+import fse from 'fs-extra';
+import fs from 'node:fs';
+import winston from 'winston';
+
+import { KVBufferStore } from '../types';
+
+export class FsKVStore implements KVBufferStore {
+  private log: winston.Logger;
+  private baseDir: string;
+  private tmpDir: string;
+
+  constructor({
+    log,
+    baseDir,
+    tmpDir,
+  }: {
+    log: winston.Logger;
+    baseDir: string;
+    tmpDir: string;
+  }) {
+    this.log = log.child({ class: this.constructor.name });
+    this.baseDir = baseDir;
+    this.tmpDir = tmpDir;
+    fs.mkdirSync(tmpDir, { recursive: true });
+  }
+
+  async get(key: string): Promise<Buffer | undefined> {
+    try {
+      if (await this.has(key)) {
+        return await fs.promises.readFile(this.bufferPath(key));
+      }
+    } catch (error: any) {
+      this.log.error('Failed to get buffer data from key/value store', {
+        key,
+        message: error.message,
+        stack: error.stack,
+      });
+    }
+    return undefined;
+  }
+
+  bufferPath(key: string): string {
+    return `${this.baseDir}/${key}`;
+  }
+
+  async has(key: string): Promise<boolean> {
+    try {
+      await fs.promises.access(this.bufferPath(key), fs.constants.F_OK);
+      return true;
+    } catch (error) {
+      return false;
+    }
+  }
+
+  async del(key: string): Promise<void> {
+    try {
+      if (await this.has(key)) {
+        await fs.promises.unlink(this.bufferPath(key));
+      }
+    } catch (error: any) {
+      this.log.error('Failed to delete buffer data from key/value store', {
+        key,
+        message: error.message,
+        stack: error.stack,
+      });
+    }
+  }
+
+  async set(key: string, buffer: Buffer): Promise<void> {
+    try {
+      if (!(await this.has(key))) {
+        // Write the block data to the temporary file in case it fails
+        const tmpPath = `${this.tmpDir}/${key}`;
+        await fs.promises.writeFile(tmpPath, buffer);
+
+        // copy the temporary file to the final location
+        await fse.move(tmpPath, this.bufferPath(key));
+      }
+    } catch (error: any) {
+      this.log.error('Failed to set buffer data in key/value store', {
+        key,
+        message: error.message,
+        stack: error.stack,
+      });
+    }
+  }
+}

src/types.d.ts

@@ -479,3 +479,10 @@ export interface MatchableItem {
 export interface ItemFilter {
   match(tx: MatchableItem): Promise<boolean>;
 }
+
+export type KVBufferStore = {
+  get(key: string): Promise<Buffer | undefined>;
+  set(key: string, buffer: Buffer): Promise<void>;
+  del(key: string): Promise<void>;
+  has(key: string): Promise<boolean>;
+};