diff --git a/package.json b/package.json index 02f4012fcc8..79e3ef0df8f 100644 --- a/package.json +++ b/package.json @@ -110,6 +110,8 @@ "test:sentinel": "jest . --passWithNoTests --forceExit -- --group=sentinel", "test:thumbnail": "jest . --passWithNoTests --forceExit -- --group=thumbnail", "test:no-mongo": "jest . --passWithNoTests --forceExit -c ./jest.config.no-mongo.ts -- --group=-mongo", + "test:hash-perf": "jest packages/protocol/packages/core/packages/hash/src/spec/PayloadHasher.bigobj.perf.spec.ts -c ./jest.config.no-mongo.ts", + "test:hash-perf-parallel": "jest packages/protocol/packages/core/packages/hash/src/spec/PayloadHasher.bigobj-parallel.perf.spec.ts -c ./jest.config.no-mongo.ts", "test:ci": "jest --runInBand --coverage --forceExit --passWithNoTests", "xyo": "node ./packages/cli/dist/cjs/index.js", "xyo-ts": "yarn workspace @xyo-network/node-cli xyo-ts", @@ -126,4 +128,4 @@ "yarn": "1.22.19" }, "type": "module" -} +} \ No newline at end of file diff --git a/packages/protocol/packages/core/packages/hash/src/PayloadHasher.ts b/packages/protocol/packages/core/packages/hash/src/PayloadHasher.ts index d2b8060295f..9300b4a7129 100644 --- a/packages/protocol/packages/core/packages/hash/src/PayloadHasher.ts +++ b/packages/protocol/packages/core/packages/hash/src/PayloadHasher.ts @@ -1,5 +1,8 @@ +import { base16 } from '@scure/base' +import { Buffer } from '@xylabs/buffer' import { AnyObject, ObjectWrapper } from '@xyo-network/object' import { WasmSupport } from '@xyo-network/wasm' +import { subtle } from 'crypto' import { sha256 } from 'hash-wasm' import shajs from 'sha.js' @@ -11,6 +14,7 @@ import { sortFields } from './sortFields' const wasmSupportStatic = new WasmSupport(['bigInt']) export class PayloadHasher extends ObjectWrapper { + static allowSubtle = true static readonly wasmInitialized = wasmSupportStatic.initialize() static readonly wasmSupport = wasmSupportStatic @@ -29,6 +33,19 @@ export class PayloadHasher extends ObjectWrappe } static async hashAsync(obj: T): Promise { + if (PayloadHasher.allowSubtle) { + try { + const enc = new TextEncoder() + const stringToHash = this.stringifyHashFields(obj) + const b = enc.encode(stringToHash) + const hashArray = await subtle.digest('SHA-256', b) + return base16.encode(Buffer.from(hashArray)).toLowerCase() + } catch (ex) { + console.log('Setting allowSubtle to false') + PayloadHasher.allowSubtle = false + } + } + await this.wasmInitialized if (this.wasmSupport.canUseWasm) { const stringToHash = this.stringifyHashFields(obj) diff --git a/packages/protocol/packages/core/packages/hash/src/spec/PayloadHasher.bigobj-parallel.perf.spec.ts b/packages/protocol/packages/core/packages/hash/src/spec/PayloadHasher.bigobj-parallel.perf.spec.ts new file mode 100644 index 00000000000..11b55828398 --- /dev/null +++ b/packages/protocol/packages/core/packages/hash/src/spec/PayloadHasher.bigobj-parallel.perf.spec.ts @@ -0,0 +1,85 @@ +/* eslint-disable max-statements */ +import { PayloadHasher } from '../PayloadHasher' + +describe('Hasher - BigObject Parallel ', () => { + const testObject = { + testArray: [1, 2, 3], + testBoolean: true, + testNull: null, + testNullObject: { t: null, x: undefined }, + testNumber: 5, + testObject: { t: 1 }, + testSomeNullObject: { s: 1, t: null, x: undefined }, + testString: 'hello there. this is a pretty long string. what do you think?', + testUndefined: undefined, + } + const bigObject = { + testArray: [1, 2, 3], + testBoolean: true, + testNull: null, + testNullObject: { t: null, x: undefined }, + testNumber: 5, + testObjArray: [testObject], + testObject: { t: 1 }, + testSomeNullObject: { s: 1, t: null, x: undefined }, + testString: 'hello there. this is a pretty long string. what do you think?', + testUndefined: undefined, + } + for (let i = 0; i < 256; i++) { + bigObject.testObjArray.push(testObject) + } + beforeAll(async () => { + await PayloadHasher.wasmInitialized + }) + + test('wasm vs js (performance-big-obj)', async () => { + const objSize = JSON.stringify(bigObject).length + const stringifyStart = Date.now() + for (let x = 0; x < 100; x++) { + JSON.stringify(bigObject) + } + const stringifyDuration = Date.now() - stringifyStart + console.log(`stringifyDuration: ${stringifyDuration}`) + console.log(`objSize: ${objSize}`) + + PayloadHasher.wasmSupport.allowWasm = false + PayloadHasher.allowSubtle = true + const subtleHashStart = Date.now() + const subtlePromises: Promise[] = [] + for (let x = 0; x < 180; x++) { + subtlePromises.push(PayloadHasher.hashAsync(bigObject)) + } + await Promise.all(subtlePromises) + const subtleHashDuration = Date.now() - subtleHashStart + console.log(`subtleHashDuration: ${subtleHashDuration} [${await PayloadHasher.hashAsync(bigObject)}]`) + + PayloadHasher.wasmSupport.allowWasm = false + PayloadHasher.allowSubtle = false + const jsHashStart = Date.now() + const jsPromises: Promise[] = [] + for (let x = 0; x < 180; x++) { + jsPromises.push(PayloadHasher.hashAsync(bigObject)) + } + await Promise.all(jsPromises) + const jsHashDuration = Date.now() - jsHashStart + console.log(`jsHashDuration: ${jsHashDuration} [${await PayloadHasher.hashAsync(bigObject)}]`) + + PayloadHasher.wasmSupport.allowWasm = true + const wasmHashStart = Date.now() + const wasmPromises: Promise[] = [] + for (let x = 0; x < 180; x++) { + wasmPromises.push(PayloadHasher.hashAsync(bigObject)) + } + await Promise.all(wasmPromises) + const wasmHashDuration = Date.now() - wasmHashStart + console.log(`wasmHashDuration: ${wasmHashDuration} [${await PayloadHasher.hashAsync(bigObject)}]`) + expect(stringifyDuration).toBeDefined() + expect(wasmHashDuration).toBeDefined() + expect(jsHashDuration).toBeDefined() + console.log( + `Wasm is ${jsHashDuration - wasmHashDuration}ms (${((1 - wasmHashDuration / jsHashDuration) * 100).toPrecision( + 2, + )}%) faster [${wasmHashDuration}ms vs ${jsHashDuration}ms ]`, + ) + }) +}) diff --git a/packages/protocol/packages/core/packages/hash/src/spec/PayloadHasher.bigobj.perf.spec.ts b/packages/protocol/packages/core/packages/hash/src/spec/PayloadHasher.bigobj.perf.spec.ts new file mode 100644 index 00000000000..e3f8db035f2 --- /dev/null +++ b/packages/protocol/packages/core/packages/hash/src/spec/PayloadHasher.bigobj.perf.spec.ts @@ -0,0 +1,134 @@ +import { PayloadHasher } from '../PayloadHasher' +/* +const cryptoTest = async () => { + const testObject = { + testArray: [1, 2, 3], + testBoolean: true, + testNull: null, + testNullObject: { t: null, x: undefined }, + testNumber: 5, + testObject: { t: 1 }, + testSomeNullObject: { s: 1, t: null, x: undefined }, + testString: 'hello there. this is a pretty long string. what do you think?', + testUndefined: undefined, + } + const bigObject = { + testArray: [1, 2, 3], + testBoolean: true, + testNull: null, + testNullObject: { t: null, x: undefined }, + testNumber: 5, + testObjArray: [testObject], + testObject: { t: 1 }, + testSomeNullObject: { s: 1, t: null, x: undefined }, + testString: 'hello there. this is a pretty long string. what do you think?', + testUndefined: undefined, + } + for (let i = 0; i < 10000; i++) { + bigObject.testObjArray.push(testObject) + } + + const objSize = JSON.stringify(bigObject).length + console.log(`objSize: ${objSize}`) + const stringifiedObj = JSON.stringify(bigObject) + const enc = new TextEncoder() + const b = enc.encode(stringifiedObj) + const cryptoHashStart = Date.now() + for (let x = 0; x < 100; x++) { + await crypto.subtle.digest('SHA-256', b) + } + + const toHexString = (bytes) => { + return Array.from(bytes, (byte) => { + return ('0' + (byte & 0xff).toString(16)).slice(-2) + }).join('') + } + + const cryptoHashDuration = Date.now() - cryptoHashStart + const result = await crypto.subtle.digest('SHA-256', b) + const dataview = new DataView(result) + const array = new Uint8Array(32) + for (let i = 0; i < dataview.byteLength; i++) { + array[i] = dataview.getUint8(i) + } + console.log(`Result: ${toHexString(array)}`) + console.log(`cryptoHashDuration: ${cryptoHashDuration}`) +} +*/ + +describe('Hasher - BigObject', () => { + const testObject = { + testArray: [1, 2, 3], + testBoolean: true, + testNull: null, + testNullObject: { t: null, x: undefined }, + testNumber: 5, + testObject: { t: 1 }, + testSomeNullObject: { s: 1, t: null, x: undefined }, + testString: 'hello there. this is a pretty long string. what do you think?', + testUndefined: undefined, + } + const bigObject = { + testArray: [1, 2, 3], + testBoolean: true, + testNull: null, + testNullObject: { t: null, x: undefined }, + testNumber: 5, + testObjArray: [testObject], + testObject: { t: 1 }, + testSomeNullObject: { s: 1, t: null, x: undefined }, + testString: 'hello there. this is a pretty long string. what do you think?', + testUndefined: undefined, + } + for (let i = 0; i < 10000; i++) { + bigObject.testObjArray.push(testObject) + } + beforeAll(async () => { + await PayloadHasher.wasmInitialized + }) + + test('wasm vs js (performance-big-obj)', async () => { + const objSize = JSON.stringify(bigObject).length + const stringifyStart = Date.now() + for (let x = 0; x < 100; x++) { + JSON.stringify(bigObject) + } + const stringifyDuration = Date.now() - stringifyStart + console.log(`stringifyDuration: ${stringifyDuration}`) + console.log(`objSize: ${objSize}`) + + PayloadHasher.wasmSupport.allowWasm = false + PayloadHasher.allowSubtle = true + const subtleHashStart = Date.now() + for (let x = 0; x < 100; x++) { + await PayloadHasher.hashAsync(bigObject) + } + const subtleHashDuration = Date.now() - subtleHashStart + console.log(`subtleHashDuration: ${subtleHashDuration} [${await PayloadHasher.hashAsync(bigObject)}]`) + + PayloadHasher.wasmSupport.allowWasm = false + PayloadHasher.allowSubtle = false + const jsHashStart = Date.now() + for (let x = 0; x < 100; x++) { + await PayloadHasher.hashAsync(bigObject) + } + const jsHashDuration = Date.now() - jsHashStart + console.log(`jsHashDuration: ${jsHashDuration} [${await PayloadHasher.hashAsync(bigObject)}]`) + + PayloadHasher.wasmSupport.allowWasm = true + const wasmHashStart = Date.now() + for (let x = 0; x < 100; x++) { + await PayloadHasher.hashAsync(bigObject) + } + const wasmHashDuration = Date.now() - wasmHashStart + console.log(`wasmHashDuration: ${wasmHashDuration} [${await PayloadHasher.hashAsync(bigObject)}]`) + expect(stringifyDuration).toBeDefined() + expect(wasmHashDuration).toBeDefined() + expect(jsHashDuration).toBeDefined() + console.log( + `Wasm is ${jsHashDuration - wasmHashDuration}ms (${((1 - wasmHashDuration / jsHashDuration) * 100).toPrecision( + 2, + )}%) faster [${wasmHashDuration}ms vs ${jsHashDuration}ms ]`, + ) + }) +}) diff --git a/packages/protocol/packages/core/packages/hash/src/spec/PayloadHasher.spec.ts b/packages/protocol/packages/core/packages/hash/src/spec/PayloadHasher.spec.ts index c09c89369c0..6c0e125f260 100644 --- a/packages/protocol/packages/core/packages/hash/src/spec/PayloadHasher.spec.ts +++ b/packages/protocol/packages/core/packages/hash/src/spec/PayloadHasher.spec.ts @@ -65,28 +65,6 @@ describe('Hasher', () => { ) }) - test('wasm vs js (performance-big-obj)', async () => { - PayloadHasher.wasmSupport.allowWasm = false - const jsHashStart = Date.now() - for (let x = 0; x < 10000; x++) { - await new PayloadHasher({ ...bigObject, nonce: x }).hashAsync() - } - const jsHashDuration = Date.now() - jsHashStart - PayloadHasher.wasmSupport.allowWasm = true - const wasmHashStart = Date.now() - for (let x = 0; x < 10000; x++) { - await new PayloadHasher({ ...bigObject, nonce: x }).hashAsync() - } - const wasmHashDuration = Date.now() - wasmHashStart - expect(wasmHashDuration).toBeDefined() - expect(jsHashDuration).toBeDefined() - console.log( - `Wasm is ${jsHashDuration - wasmHashDuration}ms (${((1 - wasmHashDuration / jsHashDuration) * 100).toPrecision( - 2, - )}%) faster [${wasmHashDuration}ms vs ${jsHashDuration}ms ]`, - ) - }) - test('wasm vs js (performance-parallel)', async () => { PayloadHasher.wasmSupport.allowWasm = false const jsTestObjects: PayloadHasher[] = [] diff --git a/tsconfig.json b/tsconfig.json index eae42704b41..868ecf43b84 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,5 +6,12 @@ "typeRoots": [] }, "exclude": [".*", "dist", "node_modules", "storybook-static", "build", "docs"], - "extends": "@xylabs/tsconfig" + "extends": "@xylabs/tsconfig", + "ts-node": { + "compilerOptions": { + "module": "ESNext" + }, + "files": true, + "transpileOnly": true + } }