From 4f41b1870169d98afa8a3948ec9fc80206f242bd Mon Sep 17 00:00:00 2001 From: David McFadzean Date: Wed, 11 Dec 2024 10:29:24 -0500 Subject: [PATCH 1/6] Added wallet cache --- packages/keymaster/package.json | 1 + packages/keymaster/src/db-wallet-cache.js | 19 +++++++++++++++++++ scripts/keychain-cli.js | 4 +++- 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 packages/keymaster/src/db-wallet-cache.js diff --git a/packages/keymaster/package.json b/packages/keymaster/package.json index dc556262..0d5d5b7d 100644 --- a/packages/keymaster/package.json +++ b/packages/keymaster/package.json @@ -8,6 +8,7 @@ "./sdk": "./src/keymaster-sdk.js", "./db/json": "./src/db-wallet-json.js", "./db/json/enc": "./src/db-wallet-json-enc.js", + "./db/cache": "./src/db-wallet-cache.js", "./db/web": "./src/db-wallet-web.js" }, "scripts": { diff --git a/packages/keymaster/src/db-wallet-cache.js b/packages/keymaster/src/db-wallet-cache.js new file mode 100644 index 00000000..06e450c0 --- /dev/null +++ b/packages/keymaster/src/db-wallet-cache.js @@ -0,0 +1,19 @@ +let actualWallet; +let cachedWallet; + +export function setWallet(wallet) { + actualWallet = wallet; +} + +export function saveWallet(wallet, overwrite = false) { + cachedWallet = null; + return actualWallet.saveWallet(wallet, overwrite); +} + +export function loadWallet() { + if (!cachedWallet) { + cachedWallet = actualWallet.loadWallet(); + } + + return cachedWallet; +} diff --git a/scripts/keychain-cli.js b/scripts/keychain-cli.js index 858726fd..cff50f42 100644 --- a/scripts/keychain-cli.js +++ b/scripts/keychain-cli.js @@ -7,6 +7,7 @@ import * as keymaster_lib from '@mdip/keymaster/lib'; import * as keymaster_sdk from '@mdip/keymaster/sdk'; import * as db_wallet_json from '@mdip/keymaster/db/json'; import * as db_wallet_enc from '@mdip/keymaster/db/json/enc'; +import * as db_wallet_cache from '@mdip/keymaster/db/cache'; import * as cipher from '@mdip/cipher/node'; dotenv.config(); @@ -1008,7 +1009,8 @@ program function getDBWallet() { if (keymasterPassphrase) { db_wallet_enc.setPassphrase(keymasterPassphrase); - return db_wallet_enc; + db_wallet_cache.setWallet(db_wallet_enc); + return db_wallet_cache; } return db_wallet_json; } From 13e3c14b490efea50b89a45df1fa9be416174b6f Mon Sep 17 00:00:00 2001 From: David McFadzean Date: Wed, 11 Dec 2024 13:15:35 -0500 Subject: [PATCH 2/6] Added KC_WALLET_CACHE env var --- docker-compose.yml | 1 + sample.env | 5 +++-- scripts/keychain-cli.js | 14 +++++++++++--- services/keymaster/server/src/config.js | 1 + services/keymaster/server/src/keymaster-api.js | 6 ++++++ 5 files changed, 22 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index fc3497f1..5df1ee1c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -45,6 +45,7 @@ services: - KC_KEYMASTER_PORT=4226 - KC_GATEKEEPER_URL=http://gatekeeper:4224 - KC_ENCRYPTED_PASSPHRASE=${KC_ENCRYPTED_PASSPHRASE} + - KC_WALLET_CACHE=${KC_WALLET_CACHE} volumes: - ./data:/app/keymaster/data user: "${KC_UID}:${KC_GID}" diff --git a/sample.env b/sample.env index 93000ecb..798eb567 100644 --- a/sample.env +++ b/sample.env @@ -11,8 +11,9 @@ KC_GATEKEEPER_REGISTRIES=hyperswarm,TBTC,TFTC KC_GATEKEEPER_PORT=4224 KC_GATEKEEPER_GC_INTERVAL=60 -# Wallet -KC_ENCRYPTED_WALLET=false +# Keymaster +KC_ENCRYPTED_PASSPHRASE="" +KC_WALLET_CACHE=false # CLI KC_GATEKEEPER_URL=http://localhost:4224 diff --git a/scripts/keychain-cli.js b/scripts/keychain-cli.js index cff50f42..fede04d1 100644 --- a/scripts/keychain-cli.js +++ b/scripts/keychain-cli.js @@ -17,6 +17,7 @@ const gatekeeperURL = process.env.KC_GATEKEEPER_URL || 'http://localhost:4224'; const keymasterURL = process.env.KC_KEYMASTER_URL; const keymasterPassphrase = process.env.KC_ENCRYPTED_PASSPHRASE; +const walletCache = process.env.KC_WALLET_CACHE ? process.env.KC_WALLET_CACHE === 'true' : false; const UPDATE_OK = "OK"; const UPDATE_FAILED = "Update failed"; @@ -1007,12 +1008,19 @@ program }); function getDBWallet() { + let wallet = db_wallet_json; + if (keymasterPassphrase) { db_wallet_enc.setPassphrase(keymasterPassphrase); - db_wallet_cache.setWallet(db_wallet_enc); - return db_wallet_cache; + wallet = db_wallet_enc; + } + + if (walletCache) { + db_wallet_cache.setWallet(wallet); + wallet = db_wallet_cache; } - return db_wallet_json; + + return wallet; } async function run() { diff --git a/services/keymaster/server/src/config.js b/services/keymaster/server/src/config.js index 50775c9c..ebc6cd9c 100644 --- a/services/keymaster/server/src/config.js +++ b/services/keymaster/server/src/config.js @@ -6,6 +6,7 @@ const config = { gatekeeperURL: process.env.KC_GATEKEEPER_URL || 'http://localhost:4224', keymasterPort: process.env.KC_KEYMASTER_PORT ? parseInt(process.env.KC_KEYMASTER_PORT) : 4226, keymasterPassphrase: process.env.KC_ENCRYPTED_PASSPHRASE, + walletCache: process.env.KC_WALLET_CACHE ? process.env.KC_WALLET_CACHE === 'true' : false, }; export default config; diff --git a/services/keymaster/server/src/keymaster-api.js b/services/keymaster/server/src/keymaster-api.js index 8536fcab..6a2ae183 100644 --- a/services/keymaster/server/src/keymaster-api.js +++ b/services/keymaster/server/src/keymaster-api.js @@ -6,6 +6,7 @@ import * as gatekeeper from '@mdip/gatekeeper/sdk'; import * as keymaster from '@mdip/keymaster/lib'; import * as wallet_json from '@mdip/keymaster/db/json'; import * as wallet_enc from '@mdip/keymaster/db/json/enc'; +import * as wallet_cache from '@mdip/keymaster/db/cache'; import * as cipher from '@mdip/cipher/node'; import config from './config.js'; const app = express(); @@ -697,6 +698,11 @@ app.listen(port, async () => { wallet = wallet_enc; } + if (config.walletCache) { + wallet_cache.setWallet(wallet); + wallet = wallet_cache; + } + await keymaster.start({ gatekeeper, wallet, cipher }); console.log(`keymaster server running on port ${port}`); From 31bf3576a321219f48fe152f58227b9803c828b5 Mon Sep 17 00:00:00 2001 From: David McFadzean Date: Wed, 11 Dec 2024 20:50:41 -0500 Subject: [PATCH 3/6] Improved cache performance --- packages/keymaster/src/db-wallet-cache.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/keymaster/src/db-wallet-cache.js b/packages/keymaster/src/db-wallet-cache.js index 06e450c0..14a887ac 100644 --- a/packages/keymaster/src/db-wallet-cache.js +++ b/packages/keymaster/src/db-wallet-cache.js @@ -6,7 +6,7 @@ export function setWallet(wallet) { } export function saveWallet(wallet, overwrite = false) { - cachedWallet = null; + cachedWallet = wallet; return actualWallet.saveWallet(wallet, overwrite); } From 08937d43175fb8edd02c55bbb240e7c50b1a3629 Mon Sep 17 00:00:00 2001 From: David McFadzean Date: Wed, 11 Dec 2024 20:50:55 -0500 Subject: [PATCH 4/6] Fix in admin script --- scripts/admin-cli.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/admin-cli.js b/scripts/admin-cli.js index 5a7459ba..ccc4a7db 100644 --- a/scripts/admin-cli.js +++ b/scripts/admin-cli.js @@ -8,7 +8,7 @@ import * as wallet from '@mdip/keymaster/db/json'; import * as cipher from '@mdip/cipher/node'; dotenv.config(); -const gatekeeperURL = process.env.KC_CLI_GATEKEEPER_URL || 'http://localhost:4224'; +const gatekeeperURL = process.env.KC_GATEKEEPER_URL || 'http://localhost:4224'; program .version('1.0.0') From 68dbe3f2bcec1dd44bac725cbde51bd4231ca516 Mon Sep 17 00:00:00 2001 From: David McFadzean Date: Thu, 12 Dec 2024 10:53:01 -0500 Subject: [PATCH 5/6] Updated satoshi-mediator to use keymaster SDK --- docker-compose.yml | 5 ++- packages/keymaster/src/db-wallet-cache.js | 8 ++-- .../keymaster/server/src/keymaster-api.js | 1 + services/mediators/satoshi/src/config.js | 1 + .../mediators/satoshi/src/satoshi-mediator.js | 40 +++++++++++++------ 5 files changed, 38 insertions(+), 17 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5df1ee1c..9d1d3d09 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -83,6 +83,7 @@ services: image: keychainmdip/satoshi-mediator environment: - KC_GATEKEEPER_URL=http://gatekeeper:4224 + - KC_KEYMASTER_URL=http://keymaster:4226 - KC_NODE_ID=${KC_NODE_ID} - KC_ENCRYPTED_PASSPHRASE=${KC_ENCRYPTED_PASSPHRASE} - KC_SAT_CHAIN=TFTC @@ -105,12 +106,12 @@ services: depends_on: - tftc-node - gatekeeper + - keymaster tbtc-node: image: keychainmdip/bitcoin-core:v28.0 volumes: - ./data/tbtc:/root/.bitcoin - tbtc-mediator: build: context: . @@ -118,6 +119,7 @@ services: image: keychainmdip/satoshi-mediator environment: - KC_GATEKEEPER_URL=http://gatekeeper:4224 + - KC_KEYMASTER_URL=http://keymaster:4226 - KC_NODE_ID=${KC_NODE_ID} - KC_ENCRYPTED_PASSPHRASE=${KC_ENCRYPTED_PASSPHRASE} - KC_SAT_CHAIN=TBTC @@ -140,6 +142,7 @@ services: depends_on: - tbtc-node - gatekeeper + - keymaster ipfs-mediator: build: diff --git a/packages/keymaster/src/db-wallet-cache.js b/packages/keymaster/src/db-wallet-cache.js index 14a887ac..247fd448 100644 --- a/packages/keymaster/src/db-wallet-cache.js +++ b/packages/keymaster/src/db-wallet-cache.js @@ -1,18 +1,18 @@ -let actualWallet; +let baseWallet; let cachedWallet; export function setWallet(wallet) { - actualWallet = wallet; + baseWallet = wallet; } export function saveWallet(wallet, overwrite = false) { cachedWallet = wallet; - return actualWallet.saveWallet(wallet, overwrite); + return baseWallet.saveWallet(wallet, overwrite); } export function loadWallet() { if (!cachedWallet) { - cachedWallet = actualWallet.loadWallet(); + cachedWallet = baseWallet.loadWallet(); } return cachedWallet; diff --git a/services/keymaster/server/src/keymaster-api.js b/services/keymaster/server/src/keymaster-api.js index 6a2ae183..6b0947c5 100644 --- a/services/keymaster/server/src/keymaster-api.js +++ b/services/keymaster/server/src/keymaster-api.js @@ -695,6 +695,7 @@ app.listen(port, async () => { if (config.keymasterPassphrase) { wallet_enc.setPassphrase(config.keymasterPassphrase); + wallet_enc.setWallet(wallet); wallet = wallet_enc; } diff --git a/services/mediators/satoshi/src/config.js b/services/mediators/satoshi/src/config.js index 67366e43..567f8f2f 100644 --- a/services/mediators/satoshi/src/config.js +++ b/services/mediators/satoshi/src/config.js @@ -5,6 +5,7 @@ dotenv.config(); const config = { nodeID: process.env.KC_NODE_ID, gatekeeperURL: process.env.KC_GATEKEEPER_URL || 'http://localhost:4224', + keymasterURL: process.env.KC_KEYMASTER_URL, keymasterPassphrase: process.env.KC_ENCRYPTED_PASSPHRASE, chain: process.env.KC_SAT_CHAIN || 'BTC', network: process.env.KC_SAT_NETWORK || 'mainnet', diff --git a/services/mediators/satoshi/src/satoshi-mediator.js b/services/mediators/satoshi/src/satoshi-mediator.js index 48e1a5e4..67a9a183 100644 --- a/services/mediators/satoshi/src/satoshi-mediator.js +++ b/services/mediators/satoshi/src/satoshi-mediator.js @@ -1,7 +1,8 @@ import fs from 'fs'; import BtcClient from 'bitcoin-core'; import * as gatekeeper from '@mdip/gatekeeper/sdk'; -import * as keymaster from '@mdip/keymaster/lib'; +import * as keymaster_lib from '@mdip/keymaster/lib'; +import * as keymaster_sdk from '@mdip/keymaster/sdk'; import * as wallet_json from '@mdip/keymaster/db/json'; import * as wallet_enc from '@mdip/keymaster/db/json/enc'; import * as cipher from '@mdip/cipher/node'; @@ -9,6 +10,7 @@ import config from './config.js'; import { InvalidParameterError } from '@mdip/common/errors'; const REGISTRY = config.chain; +let keymaster; const client = new BtcClient({ network: config.network, @@ -495,21 +497,35 @@ async function main() { return; } - await gatekeeper.start({ - url: config.gatekeeperURL, - waitUntilReady: true, - intervalSeconds: 5, - chatty: true, - }); + if (config.keymasterURL) { + keymaster = keymaster_sdk; + await keymaster.start({ + url: config.keymasterURL, + waitUntilReady: true, + intervalSeconds: 5, + chatty: true, + }); + } + else { + keymaster = keymaster_lib; + await gatekeeper.start({ + url: config.gatekeeperURL, + waitUntilReady: true, + intervalSeconds: 5, + chatty: true, + }); + + let wallet = wallet_json; - let wallet = wallet_json; + if (config.keymasterPassphrase) { + wallet_enc.setPassphrase(config.keymasterPassphrase); + wallet_enc.setWallet(wallet); + wallet = wallet_enc; + } - if (config.keymasterPassphrase) { - wallet_enc.setPassphrase(config.keymasterPassphrase); - wallet = wallet_enc; + await keymaster.start({ gatekeeper, wallet, cipher }); } - await keymaster.start({ gatekeeper, wallet, cipher }); await waitForNodeID(); if (config.importInterval > 0) { From 504c8658c845cdf63d6de9fb2ca508191866841a Mon Sep 17 00:00:00 2001 From: David McFadzean Date: Thu, 12 Dec 2024 16:17:59 -0500 Subject: [PATCH 6/6] Added createAsset to keymaster SDK --- docker-compose.yml | 1 + packages/keymaster/src/keymaster-sdk.js | 10 +++++++++ .../keymaster/server/src/keymaster-api.js | 4 ++-- .../mediators/satoshi/src/satoshi-mediator.js | 22 +++++++++---------- 4 files changed, 23 insertions(+), 14 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9d1d3d09..b5fca886 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -112,6 +112,7 @@ services: image: keychainmdip/bitcoin-core:v28.0 volumes: - ./data/tbtc:/root/.bitcoin + tbtc-mediator: build: context: . diff --git a/packages/keymaster/src/keymaster-sdk.js b/packages/keymaster/src/keymaster-sdk.js index 2de66a52..bbbfa012 100644 --- a/packages/keymaster/src/keymaster-sdk.js +++ b/packages/keymaster/src/keymaster-sdk.js @@ -316,6 +316,16 @@ export async function resolveDID(name) { } } +export async function createAsset(data, options = {}) { + try { + const response = await axios.post(`${URL}/api/v1/assets`, { data, options }); + return response.data.did; + } + catch (error) { + throwError(error); + } +} + export async function resolveAsset(name) { try { const response = await axios.get(`${URL}/api/v1/assets/${name}`); diff --git a/services/keymaster/server/src/keymaster-api.js b/services/keymaster/server/src/keymaster-api.js index 6b0947c5..0d1d11be 100644 --- a/services/keymaster/server/src/keymaster-api.js +++ b/services/keymaster/server/src/keymaster-api.js @@ -575,8 +575,8 @@ v1router.post('/schemas/:id/template/', async (req, res) => { v1router.post('/assets/', async (req, res) => { try { - const { asset, options } = req.body; - const did = await keymaster.createAsset(asset, options); + const { data, options } = req.body; + const did = await keymaster.createAsset(data, options); res.json({ did }); } catch (error) { res.status(500).send({ error: error.toString() }); diff --git a/services/mediators/satoshi/src/satoshi-mediator.js b/services/mediators/satoshi/src/satoshi-mediator.js index 67a9a183..63710ba1 100644 --- a/services/mediators/satoshi/src/satoshi-mediator.js +++ b/services/mediators/satoshi/src/satoshi-mediator.js @@ -134,8 +134,6 @@ async function importBatch(item) { return; } - console.log(JSON.stringify(item, null, 4)); - const batch = []; for (let i = 0; i < queue.length; i++) { @@ -153,8 +151,6 @@ async function importBatch(item) { }); } - // console.log(JSON.stringify(batch, null, 4)); - try { item.imported = await gatekeeper.importBatch(batch); item.processed = await gatekeeper.processEvents(); @@ -340,9 +336,10 @@ async function anchorBatch() { } const batch = await gatekeeper.getQueue(REGISTRY); - console.log(JSON.stringify(batch, null, 4)); if (batch.length > 0) { + console.log(JSON.stringify(batch, null, 4)); + const did = await keymaster.createAsset({ batch }, { registry: 'hyperswarm', controller: config.nodeID }); const txid = await createOpReturnTxn(did); @@ -369,7 +366,7 @@ async function anchorBatch() { } } else { - console.log('empty batch'); + console.log(`empty ${REGISTRY} queue`); } } @@ -497,6 +494,13 @@ async function main() { return; } + await gatekeeper.start({ + url: config.gatekeeperURL, + waitUntilReady: true, + intervalSeconds: 5, + chatty: true, + }); + if (config.keymasterURL) { keymaster = keymaster_sdk; await keymaster.start({ @@ -508,12 +512,6 @@ async function main() { } else { keymaster = keymaster_lib; - await gatekeeper.start({ - url: config.gatekeeperURL, - waitUntilReady: true, - intervalSeconds: 5, - chatty: true, - }); let wallet = wallet_json;