subtle crypto support for hasher

package.json

@@ -110,6 +110,8 @@
     "test:sentinel": "jest . --passWithNoTests --forceExit -- --group=sentinel",
     "test:thumbnail": "jest . --passWithNoTests --forceExit -- --group=thumbnail",
     "test:no-mongo": "jest . --passWithNoTests --forceExit -c ./jest.config.no-mongo.ts -- --group=-mongo",
+    "test:hash-perf": "jest packages/protocol/packages/core/packages/hash/src/spec/PayloadHasher.bigobj.perf.spec.ts -c ./jest.config.no-mongo.ts",
+    "test:hash-perf-parallel": "jest packages/protocol/packages/core/packages/hash/src/spec/PayloadHasher.bigobj-parallel.perf.spec.ts -c ./jest.config.no-mongo.ts",
     "test:ci": "jest --runInBand --coverage --forceExit --passWithNoTests",
     "xyo": "node ./packages/cli/dist/cjs/index.js",
     "xyo-ts": "yarn workspace @xyo-network/node-cli xyo-ts",
@@ -126,4 +128,4 @@
     "yarn": "1.22.19"
   },
   "type": "module"
-}
+}

packages/protocol/packages/core/packages/hash/src/PayloadHasher.ts

@@ -1,5 +1,8 @@
+import { base16 } from '@scure/base'
+import { Buffer } from '@xylabs/buffer'
 import { AnyObject, ObjectWrapper } from '@xyo-network/object'
 import { WasmSupport } from '@xyo-network/wasm'
+import { subtle } from 'crypto'
 import { sha256 } from 'hash-wasm'
 import shajs from 'sha.js'
 
@@ -11,6 +14,7 @@ import { sortFields } from './sortFields'
 const wasmSupportStatic = new WasmSupport(['bigInt'])
 
 export class PayloadHasher<T extends AnyObject = AnyObject> extends ObjectWrapper<T> {
+  static allowSubtle = true
   static readonly wasmInitialized = wasmSupportStatic.initialize()
   static readonly wasmSupport = wasmSupportStatic
 
@@ -29,6 +33,19 @@ export class PayloadHasher<T extends AnyObject = AnyObject> extends ObjectWrappe
   }
 
   static async hashAsync<T extends AnyObject>(obj: T): Promise<Hash> {
+    if (PayloadHasher.allowSubtle) {
+      try {
+        const enc = new TextEncoder()
+        const stringToHash = this.stringifyHashFields(obj)
+        const b = enc.encode(stringToHash)
+        const hashArray = await subtle.digest('SHA-256', b)
+        return base16.encode(Buffer.from(hashArray)).toLowerCase()
+      } catch (ex) {
+        console.log('Setting allowSubtle to false')
+        PayloadHasher.allowSubtle = false
+      }
+    }
+
     await this.wasmInitialized
     if (this.wasmSupport.canUseWasm) {
       const stringToHash = this.stringifyHashFields(obj)

packages/protocol/packages/core/packages/hash/src/spec/PayloadHasher.bigobj-parallel.perf.spec.ts

@@ -0,0 +1,85 @@
+/* eslint-disable max-statements */
+import { PayloadHasher } from '../PayloadHasher'
+
+describe('Hasher - BigObject Parallel ', () => {
+  const testObject = {
+    testArray: [1, 2, 3],
+    testBoolean: true,
+    testNull: null,
+    testNullObject: { t: null, x: undefined },
+    testNumber: 5,
+    testObject: { t: 1 },
+    testSomeNullObject: { s: 1, t: null, x: undefined },
+    testString: 'hello there.  this is a pretty long string.  what do you think?',
+    testUndefined: undefined,
+  }
+  const bigObject = {
+    testArray: [1, 2, 3],
+    testBoolean: true,
+    testNull: null,
+    testNullObject: { t: null, x: undefined },
+    testNumber: 5,
+    testObjArray: [testObject],
+    testObject: { t: 1 },
+    testSomeNullObject: { s: 1, t: null, x: undefined },
+    testString: 'hello there.  this is a pretty long string.  what do you think?',
+    testUndefined: undefined,
+  }
+  for (let i = 0; i < 256; i++) {
+    bigObject.testObjArray.push(testObject)
+  }
+  beforeAll(async () => {
+    await PayloadHasher.wasmInitialized
+  })
+
+  test('wasm vs js (performance-big-obj)', async () => {
+    const objSize = JSON.stringify(bigObject).length
+    const stringifyStart = Date.now()
+    for (let x = 0; x < 100; x++) {
+      JSON.stringify(bigObject)
+    }
+    const stringifyDuration = Date.now() - stringifyStart
+    console.log(`stringifyDuration: ${stringifyDuration}`)
+    console.log(`objSize: ${objSize}`)
+
+    PayloadHasher.wasmSupport.allowWasm = false
+    PayloadHasher.allowSubtle = true
+    const subtleHashStart = Date.now()
+    const subtlePromises: Promise<string>[] = []
+    for (let x = 0; x < 180; x++) {
+      subtlePromises.push(PayloadHasher.hashAsync(bigObject))
+    }
+    await Promise.all(subtlePromises)
+    const subtleHashDuration = Date.now() - subtleHashStart
+    console.log(`subtleHashDuration: ${subtleHashDuration} [${await PayloadHasher.hashAsync(bigObject)}]`)
+
+    PayloadHasher.wasmSupport.allowWasm = false
+    PayloadHasher.allowSubtle = false
+    const jsHashStart = Date.now()
+    const jsPromises: Promise<string>[] = []
+    for (let x = 0; x < 180; x++) {
+      jsPromises.push(PayloadHasher.hashAsync(bigObject))
+    }
+    await Promise.all(jsPromises)
+    const jsHashDuration = Date.now() - jsHashStart
+    console.log(`jsHashDuration: ${jsHashDuration} [${await PayloadHasher.hashAsync(bigObject)}]`)
+
+    PayloadHasher.wasmSupport.allowWasm = true
+    const wasmHashStart = Date.now()
+    const wasmPromises: Promise<string>[] = []
+    for (let x = 0; x < 180; x++) {
+      wasmPromises.push(PayloadHasher.hashAsync(bigObject))
+    }
+    await Promise.all(wasmPromises)
+    const wasmHashDuration = Date.now() - wasmHashStart
+    console.log(`wasmHashDuration: ${wasmHashDuration} [${await PayloadHasher.hashAsync(bigObject)}]`)
+    expect(stringifyDuration).toBeDefined()
+    expect(wasmHashDuration).toBeDefined()
+    expect(jsHashDuration).toBeDefined()
+    console.log(
+      `Wasm is ${jsHashDuration - wasmHashDuration}ms (${((1 - wasmHashDuration / jsHashDuration) * 100).toPrecision(
+        2,
+      )}%) faster [${wasmHashDuration}ms vs ${jsHashDuration}ms ]`,
+    )
+  })
+})

packages/protocol/packages/core/packages/hash/src/spec/PayloadHasher.bigobj.perf.spec.ts

@@ -0,0 +1,134 @@
+import { PayloadHasher } from '../PayloadHasher'
+/*
+const cryptoTest = async () => {
+  const testObject = {
+    testArray: [1, 2, 3],
+    testBoolean: true,
+    testNull: null,
+    testNullObject: { t: null, x: undefined },
+    testNumber: 5,
+    testObject: { t: 1 },
+    testSomeNullObject: { s: 1, t: null, x: undefined },
+    testString: 'hello there.  this is a pretty long string.  what do you think?',
+    testUndefined: undefined,
+  }
+  const bigObject = {
+    testArray: [1, 2, 3],
+    testBoolean: true,
+    testNull: null,
+    testNullObject: { t: null, x: undefined },
+    testNumber: 5,
+    testObjArray: [testObject],
+    testObject: { t: 1 },
+    testSomeNullObject: { s: 1, t: null, x: undefined },
+    testString: 'hello there.  this is a pretty long string.  what do you think?',
+    testUndefined: undefined,
+  }
+  for (let i = 0; i < 10000; i++) {
+    bigObject.testObjArray.push(testObject)
+  }
+
+  const objSize = JSON.stringify(bigObject).length
+  console.log(`objSize: ${objSize}`)
+  const stringifiedObj = JSON.stringify(bigObject)
+  const enc = new TextEncoder()
+  const b = enc.encode(stringifiedObj)
+  const cryptoHashStart = Date.now()
+  for (let x = 0; x < 100; x++) {
+    await crypto.subtle.digest('SHA-256', b)
+  }
+
+  const toHexString = (bytes) => {
+    return Array.from(bytes, (byte) => {
+      return ('0' + (byte & 0xff).toString(16)).slice(-2)
+    }).join('')
+  }
+
+  const cryptoHashDuration = Date.now() - cryptoHashStart
+  const result = await crypto.subtle.digest('SHA-256', b)
+  const dataview = new DataView(result)
+  const array = new Uint8Array(32)
+  for (let i = 0; i < dataview.byteLength; i++) {
+    array[i] = dataview.getUint8(i)
+  }
+  console.log(`Result: ${toHexString(array)}`)
+  console.log(`cryptoHashDuration: ${cryptoHashDuration}`)
+}
+*/
+
+describe('Hasher - BigObject', () => {
+  const testObject = {
+    testArray: [1, 2, 3],
+    testBoolean: true,
+    testNull: null,
+    testNullObject: { t: null, x: undefined },
+    testNumber: 5,
+    testObject: { t: 1 },
+    testSomeNullObject: { s: 1, t: null, x: undefined },
+    testString: 'hello there.  this is a pretty long string.  what do you think?',
+    testUndefined: undefined,
+  }
+  const bigObject = {
+    testArray: [1, 2, 3],
+    testBoolean: true,
+    testNull: null,
+    testNullObject: { t: null, x: undefined },
+    testNumber: 5,
+    testObjArray: [testObject],
+    testObject: { t: 1 },
+    testSomeNullObject: { s: 1, t: null, x: undefined },
+    testString: 'hello there.  this is a pretty long string.  what do you think?',
+    testUndefined: undefined,
+  }
+  for (let i = 0; i < 10000; i++) {
+    bigObject.testObjArray.push(testObject)
+  }
+  beforeAll(async () => {
+    await PayloadHasher.wasmInitialized
+  })
+
+  test('wasm vs js (performance-big-obj)', async () => {
+    const objSize = JSON.stringify(bigObject).length
+    const stringifyStart = Date.now()
+    for (let x = 0; x < 100; x++) {
+      JSON.stringify(bigObject)
+    }
+    const stringifyDuration = Date.now() - stringifyStart
+    console.log(`stringifyDuration: ${stringifyDuration}`)
+    console.log(`objSize: ${objSize}`)
+
+    PayloadHasher.wasmSupport.allowWasm = false
+    PayloadHasher.allowSubtle = true
+    const subtleHashStart = Date.now()
+    for (let x = 0; x < 100; x++) {
+      await PayloadHasher.hashAsync(bigObject)
+    }
+    const subtleHashDuration = Date.now() - subtleHashStart
+    console.log(`subtleHashDuration: ${subtleHashDuration} [${await PayloadHasher.hashAsync(bigObject)}]`)
+
+    PayloadHasher.wasmSupport.allowWasm = false
+    PayloadHasher.allowSubtle = false
+    const jsHashStart = Date.now()
+    for (let x = 0; x < 100; x++) {
+      await PayloadHasher.hashAsync(bigObject)
+    }
+    const jsHashDuration = Date.now() - jsHashStart
+    console.log(`jsHashDuration: ${jsHashDuration} [${await PayloadHasher.hashAsync(bigObject)}]`)
+
+    PayloadHasher.wasmSupport.allowWasm = true
+    const wasmHashStart = Date.now()
+    for (let x = 0; x < 100; x++) {
+      await PayloadHasher.hashAsync(bigObject)
+    }
+    const wasmHashDuration = Date.now() - wasmHashStart
+    console.log(`wasmHashDuration: ${wasmHashDuration} [${await PayloadHasher.hashAsync(bigObject)}]`)
+    expect(stringifyDuration).toBeDefined()
+    expect(wasmHashDuration).toBeDefined()
+    expect(jsHashDuration).toBeDefined()
+    console.log(
+      `Wasm is ${jsHashDuration - wasmHashDuration}ms (${((1 - wasmHashDuration / jsHashDuration) * 100).toPrecision(
+        2,
+      )}%) faster [${wasmHashDuration}ms vs ${jsHashDuration}ms ]`,
+    )
+  })
+})

packages/protocol/packages/core/packages/hash/src/spec/PayloadHasher.spec.ts

@@ -65,28 +65,6 @@ describe('Hasher', () => {
     )
   })
 
-  test('wasm vs js (performance-big-obj)', async () => {
-    PayloadHasher.wasmSupport.allowWasm = false
-    const jsHashStart = Date.now()
-    for (let x = 0; x < 10000; x++) {
-      await new PayloadHasher({ ...bigObject, nonce: x }).hashAsync()
-    }
-    const jsHashDuration = Date.now() - jsHashStart
-    PayloadHasher.wasmSupport.allowWasm = true
-    const wasmHashStart = Date.now()
-    for (let x = 0; x < 10000; x++) {
-      await new PayloadHasher({ ...bigObject, nonce: x }).hashAsync()
-    }
-    const wasmHashDuration = Date.now() - wasmHashStart
-    expect(wasmHashDuration).toBeDefined()
-    expect(jsHashDuration).toBeDefined()
-    console.log(
-      `Wasm is ${jsHashDuration - wasmHashDuration}ms (${((1 - wasmHashDuration / jsHashDuration) * 100).toPrecision(
-        2,
-      )}%) faster [${wasmHashDuration}ms vs ${jsHashDuration}ms ]`,
-    )
-  })
-
   test('wasm vs js (performance-parallel)', async () => {
     PayloadHasher.wasmSupport.allowWasm = false
     const jsTestObjects: PayloadHasher[] = []

tsconfig.json

@@ -6,5 +6,12 @@
     "typeRoots": []
   },
   "exclude": [".*", "dist", "node_modules", "storybook-static", "build", "docs"],
-  "extends": "@xylabs/tsconfig"
+  "extends": "@xylabs/tsconfig",
+  "ts-node": {
+    "compilerOptions": {
+      "module": "ESNext"
+    },
+    "files": true,
+    "transpileOnly": true
+  }
 }