refactor: Remove event caching from gatekeeper (#461)

* Removed eventsCache * Fix for resetDb in json-cache * Removed getEvents wrapper * Removed primeCache option * Improved coverage * Removed getAllEvents from dbs
KeychainMDIP · Dec 4, 2024 · 6bf0e72 · 6bf0e72
1 parent 75a04ab
commit 6bf0e72
Show file tree

Hide file tree

Showing 10 changed files with 19 additions and 184 deletions.
diff --git a/packages/gatekeeper/src/db-json-cache.js b/packages/gatekeeper/src/db-json-cache.js
@@ -10,6 +10,10 @@ function loadDb() {
     if (!dbCache) {
         try {
             dbCache = JSON.parse(fs.readFileSync(dbName));
+
+            if (!dbCache.dids) {
+                throw new Error();
+            }
         }
         catch (err) {
             dbCache = { dids: {} };
@@ -62,6 +66,7 @@ export async function resetDb() {
         fs.rmSync(dbName);
     }
     dbCache = null;
+    return loadDb();
 }
 
 export async function addEvent(did, event) {
@@ -83,11 +88,6 @@ export async function addEvent(did, event) {
     writeDb(db);
 }
 
-export async function getAllEvents() {
-    const db = loadDb();
-    return JSON.parse(JSON.stringify(db.dids));
-}
-
 export async function getEvents(did) {
     try {
         const db = loadDb();

diff --git a/packages/gatekeeper/src/db-json.js b/packages/gatekeeper/src/db-json.js
@@ -55,11 +55,6 @@ export async function addEvent(did, event) {
     writeDb(db);
 }
 
-export async function getAllEvents() {
-    const db = loadDb();
-    return db.dids;
-}
-
 export async function getEvents(did) {
     try {
         const db = loadDb();

diff --git a/packages/gatekeeper/src/db-mongodb.js b/packages/gatekeeper/src/db-mongodb.js
@@ -70,15 +70,6 @@ export async function getAllKeys() {
     return rows.map(row => row.id);
 }
 
-export async function getAllEvents() {
-    let allEvents = {};
-    const keys = await getAllKeys();
-    for (const key of keys) {
-        allEvents[key] = await getEvents(key);
-    }
-    return allEvents;
-}
-
 export async function queueOperation(registry, op) {
     await db.collection('queue').updateOne(
         { id: registry },

diff --git a/packages/gatekeeper/src/db-redis.js b/packages/gatekeeper/src/db-redis.js
@@ -75,15 +75,6 @@ export async function getAllKeys() {
     return keys.map(key => key.split('/').pop()); // Extract the id part from the key
 }
 
-export async function getAllEvents() {
-    let allEvents = {};
-    const keys = await getAllKeys();
-    for (const key of keys) {
-        allEvents[key] = await getEvents(key);
-    }
-    return allEvents;
-}
-
 export async function queueOperation(registry, op) {
     await redis.rpush(`${DB}/queue/${registry}`, JSON.stringify(op));
 }

diff --git a/packages/gatekeeper/src/db-sqlite.js b/packages/gatekeeper/src/db-sqlite.js
@@ -124,12 +124,3 @@ export async function getAllKeys() {
     const rows = await db.all('SELECT id FROM dids');
     return rows.map(row => row.id);
 }
-
-export async function getAllEvents() {
-    let allEvents = {};
-    const keys = await getAllKeys();
-    for (const key of keys) {
-        allEvents[key] = await getEvents(key);
-    }
-    return allEvents;
-}
diff --git a/packages/gatekeeper/src/gatekeeper-lib.js b/packages/gatekeeper/src/gatekeeper-lib.js
@@ -17,7 +17,6 @@ const validRegistries = ['local', 'hyperswarm', 'TESS', 'TBTC', 'TFTC'];
 let supportedRegistries = null;
 
 let db = null;
-let eventsCache = {};
 let eventsQueue = [];
 let eventsSeen = {};
 let verifiedDIDs = {};
@@ -27,10 +26,6 @@ const ipfs = new IPFS({ minimal: true });
 export async function start(options = {}) {
     if (options.db) {
         db = options.db;
-
-        if (options.primeCache) {
-            await primeCache();
-        }
     }
     else {
         throw new InvalidParameterError('missing options.db');
@@ -49,17 +44,6 @@ export async function stop() {
     return ipfs.stop();
 }
 
-async function primeCache() {
-    try {
-        const allEvents = await db.getAllEvents();
-        for (const key of Object.keys(allEvents)) {
-            eventsCache[`${config.didPrefix}:${key}`] = allEvents[key];
-        }
-    }
-    catch (error) {
-    }
-}
-
 export async function verifyDb(options = {}) {
     const { chatty = true } = options;
 
@@ -93,7 +77,6 @@ export async function verifyDb(options = {}) {
             }
             invalid += 1;
             await db.deleteEvents(did);
-            delete eventsCache[did];
             continue;
         }
 
@@ -106,7 +89,6 @@ export async function verifyDb(options = {}) {
                     console.log(`removing ${n}/${total} ${did} expired`);
                 }
                 await db.deleteEvents(did);
-                delete eventsCache[did];
                 expired += 1;
             }
             else {
@@ -181,7 +163,6 @@ export async function listRegistries() {
 // For testing purposes
 export async function resetDb() {
     await db.resetDb();
-    eventsCache = {};
     verifiedDIDs = {};
 }
 
@@ -456,24 +437,10 @@ export async function generateDoc(anchor) {
     return doc;
 }
 
-async function getEvents(did) {
-    let events = eventsCache[did];
-
-    if (!events) {
-        events = await db.getEvents(did);
-
-        if (events.length > 0) {
-            eventsCache[did] = events;
-        }
-    }
-
-    return copyJSON(events);
-}
-
 export async function resolveDID(did, options = {}) {
     const { atTime, atVersion, confirm = false, verify = false } = options;
 
-    const events = await getEvents(did);
+    const events = await db.getEvents(did);
 
     if (events.length === 0) {
         throw new InvalidDIDError();
@@ -586,8 +553,6 @@ export async function updateDID(operation) {
         did: operation.did
     });
 
-    delete eventsCache[operation.did];
-
     if (registry === 'local') {
         return true;
     }
@@ -639,7 +604,7 @@ export async function getDIDs(options = {}) {
 }
 
 export async function exportDID(did) {
-    return getEvents(did);
+    return db.getEvents(did);
 }
 
 export async function exportDIDs(dids) {
@@ -667,7 +632,6 @@ export async function removeDIDs(dids) {
 
     for (const did of dids) {
         await db.deleteEvents(did);
-        delete eventsCache[did];
     }
 
     return true;
@@ -684,9 +648,7 @@ async function importEvent(event) {
     }
 
     const did = event.did;
-    //console.time('getEvents');
     const currentEvents = await db.getEvents(did);
-    //console.timeEnd('getEvents');
 
     const match = currentEvents.find(item => item.operation.signature.value === event.operation.signature.value);
 
@@ -704,7 +666,6 @@ async function importEvent(event) {
             const index = currentEvents.indexOf(match);
             currentEvents[index] = event;
             await db.setEvents(did, currentEvents);
-            delete eventsCache[did];
             return true;
         }
 
@@ -715,7 +676,6 @@ async function importEvent(event) {
 
         if (ok) {
             await db.addEvent(did, event);
-            delete eventsCache[did];
             return true;
         }
         else {

diff --git a/scripts/populate-redis.js b/scripts/populate-redis.js
diff --git a/services/gatekeeper/server/src/gatekeeper-api.js b/services/gatekeeper/server/src/gatekeeper-api.js
@@ -21,7 +21,7 @@ const db = (config.db === 'sqlite') ? db_sqlite
                     : null;
 
 await db.start('mdip');
-await gatekeeper.start({ db, primeCache: true });
+await gatekeeper.start({ db });
 
 const app = express();
 const v1router = express.Router();

diff --git a/tests/gatekeeper.test.js b/tests/gatekeeper.test.js
@@ -14,7 +14,7 @@ const mockConsole = {
 
 beforeAll(async () => {
     await db_json.start('test');
-    await gatekeeper.start({ db: db_json, console: mockConsole, primeCache: false });
+    await gatekeeper.start({ db: db_json, console: mockConsole });
 });
 
 beforeEach(async () => {
@@ -134,17 +134,6 @@ describe('start', () => {
         mockFs.restore();
     });
 
-    it('should prime the cache on demand', async () => {
-        mockFs({});
-
-        const keypair = cipher.generateRandomJwk();
-        const agentOp = await createAgentOp(keypair);
-        await gatekeeper.createDID(agentOp);
-
-        await gatekeeper.stop();
-        await gatekeeper.start({ db: db_json, console: mockConsole, primeCache: true });
-    });
-
     it('should throw exception on invalid parameters', async () => {
         mockFs({});
 
@@ -401,6 +390,15 @@ describe('createDID', () => {
             expect(error.message).toBe('Invalid operation: type=mock');
         }
 
+        try {
+            const agentOp = await createAgentOp(keypair);
+            agentOp.created = 'mock';
+            await gatekeeper.createDID(agentOp);
+            throw new ExpectedExceptionError();
+        } catch (error) {
+            expect(error.message).toBe('Invalid operation: created=mock');
+        }
+
         try {
             const agentOp = await createAgentOp(keypair);
             agentOp.mdip = null;

diff --git a/tests/netki.js b/tests/netki.js
@@ -27,7 +27,7 @@ let keymaster;
 async function setup1() {
     //await db_json.start('mdip');
     await db_redis.start('mdip');
-    await gatekeeper_lib.start({ db: db_redis, primeCache: true });
+    await gatekeeper_lib.start({ db: db_redis });
     //await gatekeeper_lib.verifyDb();
     await keymaster_lib.start({ gatekeeper: gatekeeper_lib, wallet, cipher });