Merge pull request #209 from ardriveapp/dev

PE-1974: Release ArDrive Core v1.16.0

README.md

@@ -229,7 +229,7 @@ Transaction nA1stCdTkuf290k0qsqvmJ78isEC0bwgrAi3D8Cl1LU Upload Progress: 100%
 
 To avoid redundant requests to the Arweave network for immutable ArFS entity metadata, a persistent file cache is created and maintained at:
 
-```
+```shell
 Windows: <os.homedir()>/ardrive-caches/metadata
 Non-Windows: <os.homedir()>/.ardrive/caches/metadata
 ```
@@ -242,6 +242,38 @@ Cache performance is UNDEFINED for multi-process scenarios, but is presumed to b
 
 The cache can be manually cleared safely at any time that any integrating app is not in operation.
 
+### Applying Custom MetaData to ArFS File Transactions
+
+Custom metadata can be attached to ArFS File Transactions. Metadata can be applied to either the GQL tags on the MetaData Transaction, the MetaData Transaction's Data JSON, or both.
+
+All custom tags can be accessed by using by using `ArDrive` class read methods such as `getPublicFile`, `getPrivateFile`, `listPrivateFolder`, etc.
+
+```ts
+const arDrive = arDriveAnonymousFactory({});
+const fileInfo = await arDrive.getPublicFile({ fileId });
+const myMetaDataGqlTags = fileInfo.customMetaDataGqlTags;
+const myMetaDataJsonFields = fileInfo.customMetaDataJson;
+```
+
+When the custom metadata is attached to the MetaData Transaction's GQL tags, they will become visible on any Arweave GQL gateway and also third party tools that read GQL data.
+
+When these tags are added to the MetaData Transaction's Data JSON they can be read by downloading the JSON data directly from `https://arweave.net/<metadata tx id>`.
+
+To add this custom metadata to your file metadata transactions, users can pass an object containing custom tags when wrapping content to upload:
+
+```ts
+const fileToUpload = wrapFileOrFolder(
+    'path/to/file/on/system', // File or Folder Path
+    'application/custom-content-type', // Custom Content Type
+    customMetaData: { // Custom MetaData
+        metaDataJson: { ['My-Custom-Tag-Name']: 'Single-Custom-Value' },
+        metaDataGqlTags: {
+            ['Another-Custom-Tag']: ['First-Custom-Value', 'Second-Custom-Value', 'Third-Custom-Value']
+        }
+    }
+);
+```
+
 [yarn-install]: https://yarnpkg.com/getting-started/install
 [nvm-install]: https://github.com/nvm-sh/nvm#installing-and-updating
 [wsl-install]: https://code.visualstudio.com/docs/remote/wsl

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "ardrive-core-js",
-	"version": "1.15.1",
+	"version": "1.16.0",
 	"description": "ArDrive Core contains the essential back end application features to support the ArDrive CLI and Desktop apps, such as file management, Permaweb upload/download, wallet management and other common functions.",
 	"main": "./lib/exports.js",
 	"types": "./lib/exports.d.ts",

src/ardrive.test.ts

@@ -5,7 +5,7 @@ import {
 	ArFSPublicDriveTransactionData,
 	ArFSPublicFileMetadataTransactionData,
 	ArFSPublicFolderTransactionData
-} from './arfs/arfs_tx_data_types';
+} from './arfs/tx/arfs_tx_data_types';
 import { ArFSDAO } from './arfs/arfsdao';
 import { ArDriveCommunityOracle } from './community/ardrive_community_oracle';
 import { CommunityOracle } from './community/community_oracle';

src/ardrive.ts

@@ -22,7 +22,7 @@ import {
 	ArFSObjectTransactionData,
 	ArFSPublicDriveTransactionData,
 	ArFSPrivateDriveTransactionData
-} from './arfs/arfs_tx_data_types';
+} from './arfs/tx/arfs_tx_data_types';
 import { ArFSDAO } from './arfs/arfsdao';
 import { CommunityOracle } from './community/community_oracle';
 import { deriveFileKey } from './utils/crypto';
@@ -148,7 +148,14 @@ export class ArDrive extends ArDriveAnonymous {
 		super(arFsDao);
 	}
 
-	// NOTE: Presumes that there's a sufficient wallet balance
+	/**
+	 * @deprecated Sending separate layer 1 community tips is discouraged due
+	 *  to concerns with network congestion. We find it safer to place the tip
+	 *  on the layer 1 data transaction or bundle transaction. This also prevents
+	 *  separation of file data and tip payment
+	 *
+	 * @remarks Presumes that there's a sufficient wallet balance
+	 */
 	async sendCommunityTip({ communityWinstonTip, assertBalance = false }: CommunityTipParams): Promise<TipResult> {
 		const tokenHolder: ArweaveAddress = await this.communityOracle.selectTokenHolder();
 		const arTransferBaseFee = await this.priceEstimator.getBaseWinstonPriceForByteCount(new ByteCount(0));

src/arfs/arfs_builders/arfs_builders.ts

@@ -13,7 +13,13 @@ import {
 	ContentType,
 	EntityType,
 	GQLNodeInterface,
-	GQLTagInterface
+	GQLTagInterface,
+	CustomMetaDataGqlTags,
+	isCustomMetaDataJsonFields,
+	CustomMetaData,
+	CustomMetaDataJsonFields,
+	isCustomMetaDataGqlTags,
+	FeeMultiple
 } from '../../types';
 import { GatewayAPI } from '../../utils/gateway_api';
 
@@ -43,10 +49,13 @@ export abstract class ArFSMetadataEntityBuilder<T extends ArFSEntity> {
 	name?: string;
 	txId?: TransactionID;
 	unixTime?: UnixTime;
+	boost?: FeeMultiple;
 	protected readonly entityId: AnyEntityID;
 	protected readonly gatewayApi: GatewayAPI;
 	protected readonly owner?: ArweaveAddress;
 
+	customMetaData: CustomMetaData = {};
+
 	constructor({ entityId, gatewayApi, owner }: ArFSMetadataEntityBuilderParams) {
 		this.entityId = entityId;
 		this.gatewayApi = gatewayApi;
@@ -56,6 +65,10 @@ export abstract class ArFSMetadataEntityBuilder<T extends ArFSEntity> {
 	abstract getGqlQueryParameters(): GQLTagInterface[];
 	protected abstract buildEntity(): Promise<T>;
 
+	public getDataForTxID(txId: TransactionID): Promise<Buffer> {
+		return this.gatewayApi.getTxData(txId);
+	}
+
 	/**
 	 * Parses data for builder fields from either the provided GQL tags, or from a fresh request to Arweave for tag data
 	 *
@@ -107,6 +120,9 @@ export abstract class ArFSMetadataEntityBuilder<T extends ArFSEntity> {
 				case 'Unix-Time':
 					this.unixTime = new UnixTime(+value);
 					break;
+				case 'Boost':
+					this.boost = new FeeMultiple(+value);
+					break;
 				default:
 					unparsedTags.push(tag);
 					break;
@@ -116,13 +132,58 @@ export abstract class ArFSMetadataEntityBuilder<T extends ArFSEntity> {
 		return unparsedTags;
 	}
 
-	async build(node?: GQLNodeInterface): Promise<T> {
-		await this.parseFromArweaveNode(node, this.owner);
+	public async build(node?: GQLNodeInterface): Promise<T> {
+		const extraTags = await this.parseFromArweaveNode(node, this.owner);
+		this.parseCustomMetaDataFromGqlTags(extraTags);
+
 		return this.buildEntity();
 	}
 
-	getDataForTxID(txId: TransactionID): Promise<Buffer> {
-		return this.gatewayApi.getTxData(txId);
+	private parseCustomMetaDataFromGqlTags(gqlTags: GQLTagInterface[]): void {
+		const customMetaDataGqlTags: CustomMetaDataGqlTags = {};
+
+		for (const { name, value: newValue } of gqlTags) {
+			const prevValue = customMetaDataGqlTags[name];
+
+			// Accumulate any duplicated GQL tags into string[]
+			const nextValue = prevValue
+				? Array.isArray(prevValue)
+					? [...prevValue, newValue]
+					: [prevValue, newValue]
+				: newValue;
+
+			Object.assign(customMetaDataGqlTags, { [name]: nextValue });
+		}
+
+		if (!isCustomMetaDataGqlTags(customMetaDataGqlTags)) {
+			console.error(
+				`Parsed an invalid custom metadata shape from MetaData Tx GQL Tags: ${customMetaDataGqlTags}`
+			);
+			return;
+		}
+
+		if (Object.keys(customMetaDataGqlTags).length > 0) {
+			this.customMetaData.metaDataGqlTags = customMetaDataGqlTags;
+		}
+	}
+
+	protected abstract protectedDataJsonKeys: string[];
+
+	protected parseCustomMetaDataFromDataJson(dataJson: CustomMetaDataJsonFields): void {
+		if (!isCustomMetaDataJsonFields(dataJson)) {
+			console.error(`Parsed an invalid custom metadata shape from MetaData Tx Data JSON: ${dataJson}`);
+			return;
+		}
+		const dataJsonEntries = Object.entries(dataJson).filter(([key]) => !this.protectedDataJsonKeys.includes(key));
+		const customMetaDataJson: CustomMetaDataJsonFields = {};
+
+		for (const [key, val] of dataJsonEntries) {
+			Object.assign(customMetaDataJson, { [key]: val });
+		}
+
+		if (Object.keys(customMetaDataJson).length > 0) {
+			this.customMetaData.metaDataJson = customMetaDataJson;
+		}
 	}
 }
 

src/arfs/arfs_builders/arfs_drive_builders.ts

@@ -1,5 +1,5 @@
 import { driveDecrypt } from '../../utils/crypto';
-import { EntityMetaDataTransactionData, PrivateKeyData } from '../private_key_data';
+import { PrivateKeyData } from '../private_key_data';
 import {
 	CipherIV,
 	DriveKey,
@@ -9,11 +9,12 @@ import {
 	DriveAuthMode,
 	DrivePrivacy,
 	GQLNodeInterface,
-	GQLTagInterface
+	GQLTagInterface,
+	EntityMetaDataTransactionData
 } from '../../types';
 import { Utf8ArrayToStr } from '../../utils/common';
-import { fakeEntityId } from '../../utils/constants';
-import { ArFSPublicDrive, ArFSPrivateDrive, ENCRYPTED_DATA_PLACEHOLDER, ArFSDriveEntity } from '../arfs_entities';
+import { ENCRYPTED_DATA_PLACEHOLDER, fakeEntityId } from '../../utils/constants';
+import { ArFSPublicDrive, ArFSPrivateDrive, ArFSDriveEntity } from '../arfs_entities';
 import {
 	ArFSMetadataEntityBuilder,
 	ArFSMetadataEntityBuilderParams,
@@ -22,12 +23,16 @@ import {
 import { ArFSPrivateDriveKeyless } from '../../exports';
 import { GatewayAPI } from '../../utils/gateway_api';
 
-interface DriveMetaDataTransactionData extends EntityMetaDataTransactionData {
+export interface DriveMetaDataTransactionData extends EntityMetaDataTransactionData {
 	name: string;
-	rootFolderId: FolderID;
+	rootFolderId: string;
 }
 
-export class ArFSPublicDriveBuilder extends ArFSMetadataEntityBuilder<ArFSPublicDrive> {
+abstract class ArFSDriveBuilder<T extends ArFSDriveEntity> extends ArFSMetadataEntityBuilder<T> {
+	protected readonly protectedDataJsonKeys = ['name', 'rootFolderId'];
+}
+
+export class ArFSPublicDriveBuilder extends ArFSDriveBuilder<ArFSPublicDrive> {
 	drivePrivacy?: DrivePrivacy;
 	rootFolderId?: FolderID;
 
@@ -91,6 +96,8 @@ export class ArFSPublicDriveBuilder extends ArFSMetadataEntityBuilder<ArFSPublic
 				throw new Error('Invalid drive state');
 			}
 
+			this.parseCustomMetaDataFromDataJson(dataJSON);
+
 			return new ArFSPublicDrive(
 				this.appName,
 				this.appVersion,
@@ -102,15 +109,18 @@ export class ArFSPublicDriveBuilder extends ArFSMetadataEntityBuilder<ArFSPublic
 				this.txId,
 				this.unixTime,
 				this.drivePrivacy,
-				this.rootFolderId
+				this.rootFolderId,
+				this.boost,
+				this.customMetaData.metaDataGqlTags,
+				this.customMetaData.metaDataJson
 			);
 		}
 
 		throw new Error('Invalid drive state');
 	}
 }
 
-export class ArFSPrivateDriveBuilder extends ArFSMetadataEntityBuilder<ArFSPrivateDrive> {
+export class ArFSPrivateDriveBuilder extends ArFSDriveBuilder<ArFSPrivateDrive> {
 	drivePrivacy?: DrivePrivacy;
 	rootFolderId?: FolderID;
 	driveAuthMode?: DriveAuthMode;
@@ -194,7 +204,9 @@ export class ArFSPrivateDriveBuilder extends ArFSMetadataEntityBuilder<ArFSPriva
 			const decryptedDriveJSON: DriveMetaDataTransactionData = await JSON.parse(decryptedDriveString);
 
 			this.name = decryptedDriveJSON.name;
-			this.rootFolderId = decryptedDriveJSON.rootFolderId;
+			this.rootFolderId = EID(decryptedDriveJSON.rootFolderId);
+
+			this.parseCustomMetaDataFromDataJson(decryptedDriveJSON);
 
 			return new ArFSPrivateDrive(
 				this.appName,
@@ -211,7 +223,10 @@ export class ArFSPrivateDriveBuilder extends ArFSMetadataEntityBuilder<ArFSPriva
 				this.driveAuthMode,
 				this.cipher,
 				this.cipherIV,
-				this.driveKey
+				this.driveKey,
+				this.boost,
+				this.customMetaData.metaDataGqlTags,
+				this.customMetaData.metaDataJson
 			);
 		}
 
@@ -231,7 +246,7 @@ export interface SafeArFSPrivateMetadataEntityBuilderParams extends ArFSMetadata
 	privateKeyData: PrivateKeyData;
 }
 
-export class SafeArFSDriveBuilder extends ArFSMetadataEntityBuilder<ArFSDriveEntity> {
+export class SafeArFSDriveBuilder extends ArFSDriveBuilder<ArFSDriveEntity> {
 	drivePrivacy?: DrivePrivacy;
 	rootFolderId?: FolderID;
 	driveAuthMode?: DriveAuthMode;
@@ -322,7 +337,7 @@ export class SafeArFSDriveBuilder extends ArFSMetadataEntityBuilder<ArFSDriveEnt
 					if (this.cipher?.length && this.driveAuthMode?.length && this.cipherIV?.length) {
 						const placeholderDriveData = {
 							name: ENCRYPTED_DATA_PLACEHOLDER,
-							rootFolderId: new EncryptedEntityID()
+							rootFolderId: ENCRYPTED_DATA_PLACEHOLDER
 						};
 						return this.privateKeyData.safelyDecryptToJson<DriveMetaDataTransactionData>(
 							this.cipherIV,
@@ -339,7 +354,9 @@ export class SafeArFSDriveBuilder extends ArFSMetadataEntityBuilder<ArFSDriveEnt
 			})();
 
 			this.name = dataJSON.name;
-			this.rootFolderId = dataJSON.rootFolderId;
+			this.rootFolderId = EID(dataJSON.rootFolderId);
+
+			this.parseCustomMetaDataFromDataJson(dataJSON);
 
 			if (isPrivate) {
 				if (!this.driveAuthMode || !this.cipher || !this.cipherIV) {
@@ -382,7 +399,10 @@ export class SafeArFSDriveBuilder extends ArFSMetadataEntityBuilder<ArFSDriveEnt
 					this.rootFolderId,
 					this.driveAuthMode,
 					this.cipher,
-					this.cipherIV
+					this.cipherIV,
+					this.boost,
+					this.customMetaData.metaDataGqlTags,
+					this.customMetaData.metaDataJson
 				);
 			}
 			return new ArFSPublicDrive(
@@ -396,7 +416,10 @@ export class SafeArFSDriveBuilder extends ArFSMetadataEntityBuilder<ArFSDriveEnt
 				this.txId,
 				this.unixTime,
 				this.drivePrivacy,
-				this.rootFolderId
+				this.rootFolderId,
+				this.boost,
+				this.customMetaData.metaDataGqlTags,
+				this.customMetaData.metaDataJson
 			);
 		}
 		throw new Error('Invalid drive state');