From 828f971779e733f72d45276c654f747e2dacdc25 Mon Sep 17 00:00:00 2001 From: David McFadzean Date: Mon, 18 Mar 2024 12:21:31 -0400 Subject: [PATCH 1/2] Refactored mergeDb --- docker-compose.yml | 2 + gatekeeper.js | 24 +++++-- hyperswarm-mediator.js | 139 +++++++++++++++++++++++------------------ 3 files changed, 99 insertions(+), 66 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b445b0c3..b443e139 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,8 @@ services: image: macterra/gatekeeper volumes: - ./data:/app/data + ports: + - "3000:3000" hyperswarm: build: diff --git a/gatekeeper.js b/gatekeeper.js index dbafbd0e..c8191ae8 100644 --- a/gatekeeper.js +++ b/gatekeeper.js @@ -462,15 +462,29 @@ export async function importDID(txns) { } export async function mergeBatch(batch) { - let merged = 0; + let verified = 0; + let updated = 0; + let failed = 0; for (const txns of batch) { - const diff = await importDID(txns); + try { + const diff = await importDID(txns); - if (diff > 0) { - merged += 1; + if (diff > 0) { + updated += 1; + } + else { + verified += 1; + } + } + catch { + failed += 1; } } - return merged; + return { + verified: verified, + updated: updated, + failed: failed, + }; } diff --git a/hyperswarm-mediator.js b/hyperswarm-mediator.js index 383d3cc3..b5ec3b17 100644 --- a/hyperswarm-mediator.js +++ b/hyperswarm-mediator.js @@ -12,7 +12,7 @@ import config from './config.js'; import { EventEmitter } from 'events'; EventEmitter.defaultMaxListeners = 100; -const protocol = '/MDIP/v22.03.01'; +const protocol = '/MDIP/v22.03.18'; const swarm = new Hyperswarm(); const peerName = b4a.toString(swarm.keyPair.publicKey, 'hex'); @@ -37,6 +37,10 @@ function shortName(name) { return name.slice(0, 4) + '-' + name.slice(-4); } +function isEmpty(obj) { + return Object.keys(obj).length === 0 && obj.constructor === Object; +} + function loadDb() { const dbName = 'data/mdip.json'; @@ -55,13 +59,18 @@ async function shareDb() { try { const db = loadDb(); - const hash = cipher.hashJSON(db); + + if (isEmpty(db) || !db.hyperswarm || isEmpty(db.hyperswarm)) { + return; + } + + const hash = cipher.hashJSON(db.hyperswarm); messagesSeen[hash] = true; const msg = { hash: hash.toString(), - data: db, + data: db.hyperswarm, relays: [], }; @@ -90,48 +99,36 @@ async function relayDb(msg) { } } +async function mergeBatch(batch) { + try { + console.log(`mergeBatch: merging ${batch.length} DIDs...`); + const { verified, updated, failed } = await gatekeeper.mergeBatch(batch); + console.log(`* ${verified} verified, ${updated} updated, ${failed} failed`); + } + catch (error) { + console.error(`mergeBatch error: ${error}`); + } +} + async function mergeDb(db) { merging = true; - if (db.hyperswarm) { + if (db) { // Import DIDs by creation time order to avoid dependency errors - let dids = Object.keys(db.hyperswarm); - dids.sort((a, b) => db.hyperswarm[a][0].time - db.hyperswarm[b][0].time); + let dids = Object.keys(db); + dids.sort((a, b) => db[a][0].time - db[b][0].time); let batch = []; for (const did of dids) { - console.log(`Adding to batch: ${did} ${db.hyperswarm[did][0].time}`); - batch.push(db.hyperswarm[did]); + //console.log(`Adding to batch: ${did} ${db.hyperswarm[did][0].time}`); + batch.push(db[did]); if (batch.length >= 100) { - try { - const imported = await gatekeeper.mergeBatch(batch); - if (imported > 0) { - console.log(`* imported ${imported} DIDs *`); - } - else { - console.log(`* DID synchronization confirmed *`); - } - } - catch (error) { - console.error(`error importing DID: ${did}: ${error}`); - } - + await mergeBatch(batch); batch = []; } } - try { - const imported = await gatekeeper.mergeBatch(batch); - if (imported > 0) { - console.log(`* imported ${imported} DIDs *`); - } - else { - console.log(`* DID synchronization confirmed *`); - } - } - catch (error) { - console.error(`error importing DID: ${did}: ${error}`); - } + await mergeBatch(batch); } merging = false; } @@ -145,11 +142,21 @@ let queue = asyncLib.queue(async function (task, callback) { if (!seen) { messagesSeen[hash] = true; + + const db = msg.data; + + if (isEmpty(db)) { + return; + } + + // const dbName = `${hash}.json` + // fs.writeFileSync(dbName, JSON.stringify(db, null, 4)); + msg.relays.push(name); logMsg(msg.relays[0], hash); relayDb(msg); - console.log(`* merging db ${hash} *`); - await mergeDb(msg.data); + console.log(`* merging db ${shortName(hash)} *`); + await mergeDb(db); } else { console.log(`received old db: ${shortName(hash)} from: ${shortName(name)}`); @@ -173,32 +180,6 @@ function logMsg(name, hash) { console.log(`--- ${conns.length} nodes connected, ${detected} nodes detected`); } -setInterval(async () => { - try { - const version = gatekeeper.getVersion(); - - if (version) { - shareDb(); - } - } - catch (error) { - console.error(`Error: ${error}`); - } -}, 10000); - -// Join a common topic -const hash = sha256(protocol); -const networkID = Buffer.from(hash).toString('hex'); -const topic = b4a.from(networkID, 'hex'); -const discovery = swarm.join(topic, { client: true, server: true }); - -// The flushed promise will resolve when the topic has been fully announced to the DHT -discovery.flushed().then(() => { - console.log(`connecting to gatekeeper at ${config.gatekeeperURL}`); - console.log(`hyperswarm peer id: ${peerName}`); - console.log('joined topic:', b4a.toString(topic, 'hex')); -}); - process.on('uncaughtException', (error) => { //console.error('Unhandled exception caught'); console.error('Unhandled exception caught', error); @@ -214,3 +195,39 @@ process.stdin.on('data', d => { process.exit(); } }); + +// Join a common topic +const hash = sha256(protocol); +const networkID = Buffer.from(hash).toString('hex'); +const topic = b4a.from(networkID, 'hex'); + +async function start() { + console.log(`hyperswarm peer id: ${peerName}`); + console.log('joined topic:', b4a.toString(topic, 'hex')); + + setInterval(async () => { + try { + const version = gatekeeper.getVersion(); + + if (version) { + shareDb(); + } + } + catch (error) { + console.error(`Error: ${error}`); + } + }, 10000); +} + +function main() { + console.log(`connecting to gatekeeper at ${config.gatekeeperURL}`); + + const discovery = swarm.join(topic, { client: true, server: true }); + + // The flushed promise will resolve when the topic has been fully announced to the DHT + discovery.flushed().then(() => { + start(); + }); +} + +main(); From 98e491ed4759dbb691ffb9826607865446746974 Mon Sep 17 00:00:00 2001 From: David McFadzean Date: Mon, 18 Mar 2024 13:40:18 -0400 Subject: [PATCH 2/2] Added verifyDb --- gatekeeper.js | 34 +++++++++++++++++++++++++++++++- server.js | 54 +++++++++++++++++++++++++++++---------------------- 2 files changed, 64 insertions(+), 24 deletions(-) diff --git a/gatekeeper.js b/gatekeeper.js index c8191ae8..bc85c14f 100644 --- a/gatekeeper.js +++ b/gatekeeper.js @@ -32,6 +32,27 @@ export function writeDb(db) { fs.writeFileSync(dbName, JSON.stringify(db, null, 4)); } +export async function verifyDb() { + const db = loadDb(); + const dids = Object.keys(db.anchors); + let n = 0; + let invalid = 0; + + for (const did of dids) { + n += 1; + try { + const doc = await resolveDID(did, null, true); + console.log(`${n} ${did} OK`); + } + catch (error) { + console.log(`${n} ${did} ${error}`); + invalid += 1; + } + } + + return invalid; +} + let helia = null; let ipfs = null; @@ -317,7 +338,7 @@ export function fetchUpdates(registry, did) { return []; } -export async function resolveDID(did, asOfTime = null) { +export async function resolveDID(did, asOfTime = null, verify = false) { let doc = await generateDoc(did); let mdip = doc?.didDocumentMetadata?.mdip; @@ -345,6 +366,9 @@ export async function resolveDID(did, asOfTime = null) { if (hash !== txn.prev) { // hash mismatch + // if (verify) { + // throw "Invalid hash"; + // } // !!! This fails on key rotation #3 (!?), disabling for now // continue; } @@ -352,6 +376,10 @@ export async function resolveDID(did, asOfTime = null) { const valid = await verifyUpdate(txn, doc); if (!valid) { + if (verify) { + throw "Invalid update"; + } + continue; } @@ -372,6 +400,10 @@ export async function resolveDID(did, asOfTime = null) { doc.didDocumentMetadata.updated = time; } else { + if (verify) { + throw "Invalid operation"; + } + console.error(`unknown op ${txn.op}`); } } diff --git a/server.js b/server.js index 77a9c624..14f20160 100644 --- a/server.js +++ b/server.js @@ -43,27 +43,27 @@ app.get('/did/:did', async (req, res) => { app.get('/explore/:did', async (req, res) => { try { - const doc = await gatekeeper.resolveDID(req.params.did, req.query.asof); - var hthead = ''; - hthead = hthead + '

MDIP Network Explorer

'; - hthead = hthead + ''; - var htdoc = JSON.stringify(doc,null,5); - htdoc = htdoc.replace(/"didDocument"/g, '"didDocument"'); - htdoc = htdoc.replace(/"didDocumentMetadata"/g, '"didDocumentMetadata"'); - htdoc = htdoc.replace(/"manifest"/g, '"manifest"'); - htdoc = htdoc.replace(/"issuer"/g, '"issuer"'); - htdoc = htdoc.replace(/"signer"/g, '"signer"'); - htdoc = htdoc.replace(/"id"/g, '"id"'); - htdoc = htdoc.replace(/"credential"/g, '"credential"'); - htdoc = htdoc.replace(/"vault"/g, '"vault"'); - htdoc = htdoc.replace(/"(did:mdip:.*)"/g, '"$1"'); - htdoc = hthead + ''; - var now = new Date(); - htdoc = htdoc + '

' + req.params.did + '

' + htdoc + '
' + now + ''; - res.send(htdoc); - } catch (error ) { - console.error(error); - res.status(500).send(error.toString()); + const doc = await gatekeeper.resolveDID(req.params.did, req.query.asof); + var hthead = ''; + hthead = hthead + '

MDIP Network Explorer

'; + hthead = hthead + ''; + var htdoc = JSON.stringify(doc, null, 5); + htdoc = htdoc.replace(/"didDocument"/g, '"didDocument"'); + htdoc = htdoc.replace(/"didDocumentMetadata"/g, '"didDocumentMetadata"'); + htdoc = htdoc.replace(/"manifest"/g, '"manifest"'); + htdoc = htdoc.replace(/"issuer"/g, '"issuer"'); + htdoc = htdoc.replace(/"signer"/g, '"signer"'); + htdoc = htdoc.replace(/"id"/g, '"id"'); + htdoc = htdoc.replace(/"credential"/g, '"credential"'); + htdoc = htdoc.replace(/"vault"/g, '"vault"'); + htdoc = htdoc.replace(/"(did:mdip:.*)"/g, '"$1"'); + htdoc = hthead + ''; + var now = new Date(); + htdoc = htdoc + '

' + req.params.did + '

' + htdoc + '
' + now + ''; + res.send(htdoc); + } catch (error) { + console.error(error); + res.status(500).send(error.toString()); } }); @@ -123,8 +123,16 @@ app.post('/merge', async (req, res) => { const port = 3000; -app.listen(port, () => { - console.log(`Server is running on port ${port}`); +gatekeeper.verifyDb().then((invalid) => { + if (invalid === 0) { + app.listen(port, () => { + console.log(`Server is running on port ${port}`); + }); + } + else { + console.log(`${invalid} invalid DIDs in MDIP db`); + process.exit(); + } }); process.on('uncaughtException', (error) => {