updated PayloadBuilder to not wipe $meta from BoundWitnessBuilder

packages/protocol/packages/account/packages/account-model/src/Account.ts

@@ -24,7 +24,7 @@ export type AccountConfig = InitializationConfig & AccountOptions
 
 export interface AccountInstance extends KeyPairInstance {
   address: string
-  addressBytes: ArrayBuffer | undefined
+  addressBytes: ArrayBuffer
   previousHash: string | undefined
   previousHashBytes: ArrayBuffer | undefined
   sign: (hash: ArrayBuffer, previousHash: ArrayBuffer | undefined) => ArrayBuffer | Promise<ArrayBuffer>

packages/protocol/packages/boundwitness/packages/builder/src/Builder.ts

@@ -1,39 +1,50 @@
 import { toArrayBuffer, toUint8Array } from '@xylabs/arraybuffer'
 import { assertEx } from '@xylabs/assert'
 import { Address, Hash, hexFromArrayBuffer } from '@xylabs/hex'
-import { Logger } from '@xylabs/logger'
+import { JsonObject } from '@xylabs/object'
 import { AccountInstance } from '@xyo-network/account-model'
 import { BoundWitness, BoundWitnessSchema } from '@xyo-network/boundwitness-model'
 import { sortFields } from '@xyo-network/hash'
-import { PayloadBuilder, PayloadWrapper } from '@xyo-network/payload'
+import { PayloadBuilder, PayloadBuilderBase, PayloadBuilderOptions, PayloadWrapper } from '@xyo-network/payload'
 import { ModuleError, Payload, Schema, WithMeta } from '@xyo-network/payload-model'
 import { Mutex } from 'async-mutex'
 
-export interface BoundWitnessBuilderConfig {
-  /** Whether or not the payloads should be included in the metadata sent to and recorded by the ArchivistApi */
-  /** @deprecated We will be removing support for inlinePayloads soon */
-  readonly inlinePayloads?: boolean
-  /** @deprecated We will be removing support for meta soon */
-  readonly meta?: boolean
-  /** @deprecated We will be removing support for timestamp soon */
-  readonly timestamp?: boolean
+type GeneratedBoundWitnessFields = 'addresses' | 'payload_hashes' | 'payload_schemas' | 'previous_hashes'
+
+export interface BoundWitnessBuilderOptions<T extends BoundWitness = BoundWitness, TPayload extends Payload = Payload>
+  extends Omit<PayloadBuilderOptions<Omit<T, GeneratedBoundWitnessFields>>, 'schema'> {
+  readonly accounts?: AccountInstance[]
+  readonly payloadHashes?: T['payload_hashes']
+  readonly payloadSchemas?: T['payload_schemas']
+  readonly payloads?: TPayload[]
+  readonly sourceQuery?: Hash
+  readonly timestamp?: boolean | number
 }
 
-export class BoundWitnessBuilder<TBoundWitness extends BoundWitness<{ schema: string }> = BoundWitness, TPayload extends Payload = Payload> {
+export class BoundWitnessBuilder<T extends BoundWitness = BoundWitness, TPayload extends Payload = Payload> extends PayloadBuilderBase<
+  Omit<T, GeneratedBoundWitnessFields>,
+  BoundWitnessBuilderOptions<T> & { schema: BoundWitnessSchema }
+> {
   private static readonly _buildMutex = new Mutex()
-  private _accounts: AccountInstance[] = []
-  private _errorHashes: string[] | undefined
+  private _accounts: AccountInstance[]
+  private _errorHashes?: string[]
   private _errors: ModuleError[] = []
-  private _payloadHashes: string[] | undefined
-  private _payloadSchemas: string[] | undefined
-  private _payloads: TPayload[] = []
-  private _sourceQuery: string | undefined
-  private _timestamp = Date.now()
-
-  constructor(
-    readonly config: BoundWitnessBuilderConfig = { inlinePayloads: false },
-    protected readonly logger?: Logger,
-  ) {}
+  private _payloadHashes?: string[]
+  private _payloadSchemas?: string[]
+  private _payloads: TPayload[]
+  private _sourceQuery?: Hash
+  private _timestamp: boolean | number
+
+  constructor(options?: BoundWitnessBuilderOptions<T, TPayload>) {
+    super({ ...options, schema: BoundWitnessSchema })
+    const { accounts, payloadHashes, payloadSchemas, payloads, sourceQuery, timestamp } = options ?? {}
+    this._accounts = accounts ?? []
+    this._payloadHashes = payloadHashes
+    this._payloadSchemas = payloadSchemas
+    this._payloads = payloads ?? []
+    this._sourceQuery = sourceQuery
+    this._timestamp = timestamp ?? true
+  }
 
   private get _payload_schemas(): string[] {
     return (
@@ -60,30 +71,30 @@ export class BoundWitnessBuilder<TBoundWitness extends BoundWitness<{ schema: st
     return payload.$meta.signatures[this.addressIndex(payload, address)]
   }
 
-  async build(meta = false): Promise<[WithMeta<TBoundWitness>, TPayload[], ModuleError[]]> {
-    if (meta) {
-      console.log('BoundWitnessBuilder: Calling build with meta=true will be disallowed soon')
-    }
+  async build(): Promise<[WithMeta<T>, TPayload[], ModuleError[]]> {
     return await BoundWitnessBuilder._buildMutex.runExclusive(async () => {
       const hashableFields = await this.hashableFields()
       const hash = (await PayloadBuilder.build(hashableFields)).$hash
 
       /* get all the previousHashes to verify atomic signing */
       const previousHashes = this._accounts.map((account) => account.previousHash)
 
-      const meta =
-        hashableFields.addresses.length > 0
-          ? {
-              $meta: {
-                signatures: await this.signatures(hash, previousHashes),
-              },
-            }
-          : {}
+      const metaHolder: { $meta?: JsonObject } = {}
+
+      if (hashableFields.addresses.length > 0) {
+        metaHolder.$meta = metaHolder.$meta ?? {}
+        metaHolder.$meta.signatures = await this.signatures(hash, previousHashes)
+      }
 
-      const ret: WithMeta<TBoundWitness> = {
+      if (this._sourceQuery) {
+        metaHolder.$meta = metaHolder.$meta ?? {}
+        metaHolder.$meta.sourceQuery = this._sourceQuery
+      }
+
+      const ret: WithMeta<T> = {
         ...hashableFields,
         $hash: hash,
-        ...meta,
+        ...metaHolder,
       }
       return [ret, this._payloads, this._errors]
     })
@@ -111,33 +122,29 @@ export class BoundWitnessBuilder<TBoundWitness extends BoundWitness<{ schema: st
     return this
   }
 
-  async hashableFields(): Promise<TBoundWitness> {
-    const addresses = this._accounts.map((account) =>
-      account.addressBytes ? hexFromArrayBuffer(account.addressBytes, { prefix: false }) : undefined,
-    )
+  override async hashableFields(): Promise<T> {
+    const addresses = this._accounts.map((account) => hexFromArrayBuffer(account.addressBytes, { prefix: false }))
     const previous_hashes = this._accounts.map((account) => account.previousHash ?? null)
     const payload_hashes = assertEx(await this.getPayloadHashes(), 'Missing payload_hashes')
     const payload_schemas = assertEx(this._payload_schemas, 'Missing payload_schemas')
-    const result: TBoundWitness = {
-      addresses: assertEx(addresses, 'Missing addresses'),
+    const result: T = {
+      ...(await super.hashableFields()),
+      addresses,
       payload_hashes,
       payload_schemas,
       previous_hashes,
-      schema: BoundWitnessSchema,
-    } as TBoundWitness
+    } as T
 
     assertEx(result.payload_hashes?.length === result.payload_schemas?.length, 'Payload hash/schema mismatch')
 
     assertEx(!result.payload_hashes.some((hash) => !hash), () => 'nulls found in hashes')
 
     assertEx(!result.payload_schemas.some((schema) => !schema), 'nulls found in schemas')
 
-    if (this.config.timestamp ?? true) {
+    if (typeof this._timestamp === 'number') {
       result.timestamp = this._timestamp
-    }
-
-    if (this._sourceQuery) {
-      result.sourceQuery = this._sourceQuery
+    } else if (this._timestamp) {
+      result.timestamp = Date.now()
     }
 
     return result
@@ -171,8 +178,8 @@ export class BoundWitnessBuilder<TBoundWitness extends BoundWitness<{ schema: st
     return this
   }
 
-  sourceQuery(hash?: Hash) {
-    this._sourceQuery = hash
+  sourceQuery(query?: Hash) {
+    this._sourceQuery = query
     return this
   }
 

packages/protocol/packages/boundwitness/packages/builder/src/spec/Builder.spec.ts

@@ -63,18 +63,10 @@ describe('BoundWitnessBuilder', () => {
         }
       })
     })
-    describe('with inlinePayloads true', () => {
-      it('contains the _payloads field', async () => {
-        const address = await Account.fromPhrase('sibling split sadness nose fever umbrella favorite ritual movie zone buyer movie')
-        const builder = await new BoundWitnessBuilder({ inlinePayloads: true }).witness(address).payload(payload1)
-        const [actual] = await builder.build()
-        expect(actual).toBeDefined()
-      })
-    })
-    describe('with inlinePayloads false', () => {
+    describe('with payloads', () => {
       it('omits the _payloads field', async () => {
         const address = await Account.fromPhrase('canyon defense similar chalk good box quote miss decorate load amused gown')
-        const builder = await new BoundWitnessBuilder({ inlinePayloads: false }).witness(address).payload(payload1)
+        const builder = await new BoundWitnessBuilder().witness(address).payload(payload1)
         const [actual] = await builder.build()
         expect(actual).toBeDefined()
       })

packages/protocol/packages/payload/packages/builder/src/Builder.ts

@@ -1,31 +1,15 @@
-import { assertEx } from '@xylabs/assert'
 import { Hash } from '@xylabs/hex'
-import { AnyObject, JsonObject } from '@xylabs/object'
-import { deepOmitPrefixedFields, PayloadHasher, removeEmptyFields } from '@xyo-network/hash'
+import { AnyObject } from '@xylabs/object'
+import { PayloadHasher } from '@xyo-network/hash'
 import { Payload, PayloadWithMeta, WithMeta } from '@xyo-network/payload-model'
 
-export interface PayloadBuilderOptions<T> {
-  fields?: Partial<T>
-  meta?: JsonObject
-  schema: string
-}
-
-export class PayloadBuilder<T extends Payload = Payload<AnyObject>> {
-  private _$meta?: JsonObject
-  private _fields: Partial<T> = {}
-  private _schema: string
-
-  constructor({ schema, fields, meta }: PayloadBuilderOptions<T>) {
-    this._schema = schema
-    this._fields = fields ?? {}
-    this._$meta = meta
-  }
-
-  get schema() {
-    this._schema = this._schema ?? this._fields['schema']
-    return this._schema
-  }
+import { PayloadBuilderBase } from './BuilderBase'
+import { PayloadBuilderOptions } from './Options'
 
+export class PayloadBuilder<
+  T extends Payload = Payload<AnyObject>,
+  O extends PayloadBuilderOptions<T> = PayloadBuilderOptions<T>,
+> extends PayloadBuilderBase<T, O> {
   static async build<T extends Payload>(payload: T) {
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
     const { schema, $hash, $meta, ...fields } = payload as WithMeta<T>
@@ -122,13 +106,8 @@ export class PayloadBuilder<T extends Payload = Payload<AnyObject>> {
     return result
   }
 
-  $meta(meta?: JsonObject) {
-    this._$meta = meta
-    return this
-  }
-
   async build(): Promise<WithMeta<T>> {
-    const dataHashableFields = this.dataHashableFields()
+    const dataHashableFields = await this.dataHashableFields()
     const $hash = await PayloadBuilder.hash(dataHashableFields)
     const hashableFields: PayloadWithMeta = { ...dataHashableFields, $hash }
 
@@ -138,22 +117,4 @@ export class PayloadBuilder<T extends Payload = Payload<AnyObject>> {
     }
     return hashableFields as WithMeta<T>
   }
-
-  dataHashableFields() {
-    return {
-      ...removeEmptyFields(deepOmitPrefixedFields(deepOmitPrefixedFields(this._fields, '$'), '_')),
-      schema: assertEx(this.schema, 'Payload: Missing Schema'),
-    } as T
-  }
-
-  fields(fields?: Partial<T>) {
-    if (fields) {
-      const { $meta } = fields as Partial<WithMeta<T>>
-      if ($meta) {
-        this.$meta($meta)
-      }
-      this._fields = { ...this._fields, ...removeEmptyFields(fields) }
-    }
-    return this
-  }
 }

packages/protocol/packages/payload/packages/builder/src/BuilderBase.ts

@@ -0,0 +1,57 @@
+import { assertEx } from '@xylabs/assert'
+import { AnyObject, JsonObject } from '@xylabs/object'
+import { Promisable } from '@xylabs/promise'
+import { deepOmitPrefixedFields, removeEmptyFields } from '@xyo-network/hash'
+import { Payload, Schema, WithMeta } from '@xyo-network/payload-model'
+
+import { PayloadBuilderOptions } from './Options'
+
+export class PayloadBuilderBase<T extends Payload = Payload<AnyObject>, O extends PayloadBuilderOptions<T> = PayloadBuilderOptions<T>> {
+  protected _$meta?: JsonObject
+  protected _fields?: Omit<T, 'schema' | '$hash' | '$meta'>
+  protected _schema: Schema
+
+  constructor(readonly options: O) {
+    const { schema, fields, meta } = options
+    this._schema = schema
+    this._fields = fields
+    this._$meta = meta
+  }
+
+  $meta(meta?: JsonObject) {
+    this._$meta = meta ?? (this._fields as WithMeta<T>).$meta
+    return this
+  }
+
+  async dataHashableFields(): Promise<T> {
+    return deepOmitPrefixedFields(await this.hashableFields(), '$')
+  }
+
+  //we do not require sending in $hash since it will be generated anyway
+  fields(fields: Omit<WithMeta<T>, '$hash' | 'schema'> & Partial<Pick<WithMeta<T>, '$hash' | 'schema'>>) {
+    if (fields) {
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      const { $meta, $hash, schema, ...fieldsOnly } = fields as WithMeta<T>
+      if ($meta) {
+        this.$meta($meta)
+      }
+      if (schema) {
+        this.schema(schema)
+      }
+      this._fields = { ...this._fields, ...removeEmptyFields(fieldsOnly) }
+    }
+    return this
+  }
+
+  hashableFields(): Promisable<T> {
+    const schema = assertEx(this._schema, 'Payload: Missing Schema')
+    return {
+      ...removeEmptyFields(deepOmitPrefixedFields(this._fields ?? {}, '_')),
+      schema,
+    } as T
+  }
+
+  schema(value: Schema) {
+    this._schema = value
+  }
+}

packages/protocol/packages/payload/packages/builder/src/Options.ts

@@ -0,0 +1,10 @@
+import { Logger } from '@xylabs/logger'
+import { JsonObject } from '@xylabs/object'
+import { Schema } from '@xyo-network/payload-model'
+
+export interface PayloadBuilderOptions<T> {
+  readonly fields?: Omit<T, 'schema' | '$hash' | '$meta'>
+  readonly logger?: Logger
+  readonly meta?: JsonObject
+  readonly schema: Schema
+}

packages/protocol/packages/payload/packages/builder/src/index.ts

@@ -1 +1,3 @@
 export * from './Builder'
+export * from './BuilderBase'
+export * from './Options'