Feat 52: Import and Export Bucketizer State (#53)

* bump: bump dependency version

* bump: bump dependency version

* feat: export and import Bucketizer state, and change State object

* dist: nw dist

* feat: pass null instead of ' ' when no previous state

* dist: new dist

package.json

@@ -37,8 +37,8 @@
     "@rdfjs/data-model": "^1.3.3",
     "@rdfjs/types": "^1.0.1",
     "@treecg/actor-init-ldes-client": "^2.5.7",
-    "@treecg/bucketizers": "^1.0.3",
-    "@treecg/types": "0.0.12",
+    "@treecg/bucketizers": "^1.0.4",
+    "@treecg/types": "0.0.16",
     "date-fns": "^2.23.0",
     "eslint": "^7.32.0",
     "eslint-config-es": "^3.30.15",

src/data-source-strategy/LDESClientDatasource.ts

@@ -1,8 +1,9 @@
-import type { EventStream, State } from '@treecg/actor-init-ldes-client';
+import type { EventStream } from '@treecg/actor-init-ldes-client';
 import { newEngine } from '@treecg/actor-init-ldes-client';
 import type { Member, Bucketizer } from '@treecg/types';
 import type { Config } from '../utils/Config';
 import type Datasource from '../utils/interfaces/Datasource';
+import type { LDESActionState } from '../utils/State';
 import { loadState, saveState } from '../utils/State';
 
 class LDESClientDatasource implements Datasource {
@@ -31,7 +32,7 @@ class LDESClientDatasource implements Datasource {
         });
 
         ldes.on('pause', () => {
-          saveState(ldes.exportState(), config.storage);
+          saveState(ldes.exportState(), bucketizer.exportState(), config.storage);
 
           console.log('No more data!');
           resolve(data);
@@ -52,12 +53,12 @@ class LDESClientDatasource implements Datasource {
     };
 
     const LDESClient = newEngine();
-    const state: State | null = loadState(storage);
+    const state: LDESActionState | null = loadState(storage);
     if (state === null) {
       return LDESClient.createReadStream(url, options);
     }
 
-    return LDESClient.createReadStream(url, options, state);
+    return LDESClient.createReadStream(url, options, state.LDESClientState);
   }
 }
 

src/data.ts

@@ -79,7 +79,7 @@ export class Data {
 
       ldes.on('pause', async () => {
         console.log('Export LDES Client State');
-        saveState(ldes.exportState(), this.config.storage);
+        saveState(ldes.exportState(), bucketizer.exportState(), this.config.storage);
 
         await Promise.all(tasks);
 

src/fragment-strategy/BucketizerFragmentStrategy.ts

@@ -7,6 +7,8 @@ import { DataFactory } from 'rdf-data-factory';
 import type { Config } from '../utils/Config';
 import type FragmentStrategy from '../utils/interfaces/FragmentStrategy';
 import { sanitize } from '../utils/Sanitizer';
+import type { LDESActionState } from '../utils/State';
+import { loadState } from '../utils/State';
 
 class BucketizerFragmentStrategy implements FragmentStrategy {
   private readonly bucketNode: RDF.NamedNode;
@@ -23,20 +25,22 @@ class BucketizerFragmentStrategy implements FragmentStrategy {
       pageSize: config.fragmentation_page_size,
     };
 
+    const state: LDESActionState | null = loadState(config.storage);
+
     switch (config.fragmentation_strategy) {
       case 'substring':
         console.log(`[BucketizerFragmentStrategy]: setting strategy to substrings.`);
-        return SubstringBucketizer.build(bucketizerOptions);
+        return SubstringBucketizer.build(bucketizerOptions, state === null ? null : state.BucketizerState);
 
       case 'subject-page':
         console.log(`[BucketizerFragmentStrategy]: setting strategy to subject pages.`);
-        return SubjectPageBucketizer.build(bucketizerOptions);
+        return SubjectPageBucketizer.build(bucketizerOptions, state === null ? null : state.BucketizerState);
 
       case 'basic':
       default:
 
         console.log(`[BucketizerFragmentStrategy]: setting strategy to basic.`);
-        return BasicBucketizer.build(bucketizerOptions);
+        return BasicBucketizer.build(bucketizerOptions, state === null ? null : state.BucketizerState);
     }
   }
 

src/utils/State.ts

@@ -1,18 +1,27 @@
 import { existsSync, mkdirSync, writeFileSync, readFileSync } from 'fs';
 import type { State } from '@treecg/actor-init-ldes-client';
 
-export const saveState = (state: State, storage: string): void => {
+export const saveState = (clientState: State, buckerizerState: any, storage: string): void => {
   const folder = `./.ldes/${storage}`;
   if (!existsSync(folder)) {
     mkdirSync(folder, { recursive: true });
   }
+  const state: LDESActionState = {
+    LDESClientState: clientState,
+    BucketizerState: buckerizerState,
+  };
   writeFileSync(`${folder}/state.json`, JSON.stringify(state));
 };
 
-export const loadState = (storage: string): State | null => {
+export const loadState = (storage: string): LDESActionState | null => {
   const folder = `./.ldes/${storage}`;
   if (existsSync(`${folder}/state.json`)) {
     return JSON.parse(readFileSync(`${folder}/state.json`).toString());
   }
   return null;
 };
+
+export interface LDESActionState {
+  LDESClientState: State;
+  BucketizerState: any;
+}