Merge pull request #303 from ardriveapp/dev

PE-1975: Release ArDrive CLI v1.18.0

README.md

@@ -130,6 +130,8 @@ ardrive upload-file --wallet-file /path/to/my/wallet.json --parent-folder-id "f0
         16. [Hosting a Webpage with Manifest](#hosting-a-webpage-with-manifest)
         17. [Uploading With a Custom Content Type](#custom-content-type)
         18. [Uploading a Custom Manifest](#custom-manifest)
+        19. [Uploading Files with Custom MetaData](#uploading-files-with-custom-metadata)
+        20. [Applying Unique Custom MetaData During Bulk Workflows](#applying-unique-custom-metadata-during-bulk-workflows)
     8. [Other Utility Operations](#other-utility-operations)
         1. [Monitoring Transactions](#monitoring-transactions)
         2. [Dealing With Network Congestion](#dealing-with-network-congestion)
@@ -1162,6 +1164,117 @@ https://arweave.net/{dataTxId}/custom-file-1
 https://arweave.net/{dataTxId}/custom-file-2
 ```
 
+### Uploading Files with Custom MetaData
+
+ArDrive CLI has the capability of attaching custom metadata to ArFS File and Folder MetaData Transactions during the `upload-file` command. This metadata can be applied to either the GQL tags on the MetaData Transaction and/or into the MetaData Transaction's Data JSON.
+
+All custom metadata applied must ultimately adhere to the following JSON shapes:
+
+```ts
+// GQL Tags
+type CustomMetaDataGqlTags = Record<string, string | string[]>;
+
+// Data JSON Fields
+type CustomMetaDataJsonFields = Record<string, JsonSerializable>;
+
+export type JsonSerializable =
+    | string
+    | number
+    | boolean
+    | null
+    | { [member: string]: JsonSerializable }
+    | JsonSerializable[];
+```
+
+e.g:
+
+```shell
+{ IPFS-Add: 'MY_HASH' }
+# or
+{ 'Custom Name': ['Val 1', 'Val 2'] }
+```
+
+When the custom metadata is attached to the MetaData Transaction's GQL tags, they will become visible on any Arweave GQL gateway and also third party tools that read GQL data.
+
+When these tags are added to the MetaData Transaction's Data JSON they can be read by downloading the JSON data directly from `https://arweave.net/METADATA_TX_ID`.
+
+To add this custom metadata to your file metadata transactions, CLI users can pass custom metadata these parameters:
+
+-   `--metadata-file path/to/json/schema`
+-   `--metadata-json '{"key": "val", "key-2": true, "key-3": 420, "key-4": ["more", 1337]}'`
+-   `--metadata-gql-tags "Tag-Name" "Tag Val"`
+
+The `--metadata-file` will accept a file path to JSON file containing custom metadata:
+
+```shell
+ardrive upload-file --metadata-file path/to/metadata/json # ...
+```
+
+This JSON schema object must contain instructions on where to put this metadata with the `metaDataJson` and `metaDataGqlTags` keys. e.g:
+
+```json
+{
+    "metaDataJson": {
+        "Tag-Name": ["Value-1", "Value-2"]
+    },
+    "metaDataGqlTags": {
+        "GQL Tag Name": "Tag Value"
+    }
+}
+```
+
+The `--metadata-gql-tags` parameter accepts an array of string values to be applied to the MetaData Tx GQL Tags. This method of CLI input does not support multiple tag values for a given tag name and the input must be an EVEN number of string values. (Known bug: String values starting with the `"-"` character are currently not supported. Use --metadata-file parameter instead.) e.g:
+
+```shell
+upload-file --metadata-gql-tags "Custom Tag Name" "Custom Value" # ...
+```
+
+And the `--metadata-json` parameter will accept a stringified JSON input. It will apply all declared JSON fields directly to the MetaData Tx's Data JSON. e.g:
+
+```shell
+ upload-file --metadata-json ' { "json field": "value", "another fields": false } ' # ...
+```
+
+Custom metadata applied to files and/or folders during the `upload-file` command will be read back through all existing read commands. e.g:
+
+```shell
+ardrive file-info -f 067c4008-9cbe-422e-b697-05442f73da2b
+{
+    "appName": "ArDrive-CLI",
+    "appVersion": "1.17.0",
+    "arFS": "0.11",
+    "contentType": "application/json",
+    "driveId": "967215ca-a489-494b-97ec-0dd428d7be34",
+    "entityType": "file",
+    "name": "unique-name-9718",
+    "txId": "sxg8bNu6_bbaHkJTxAINVVoz_F-LiFe6s7OnxzoJJk4",
+    "unixTime": 1657655070,
+    "size": 262148,
+    "lastModifiedDate": 1655409872705,
+    "dataTxId": "ublZcIff77ejl3m0uEA8lXEfnTWmSBOFoz-HibqKeyk",
+    "dataContentType": "text/plain",
+    "parentFolderId": "97bc4fb5-aca4-4ffe-938f-1285153d98ca",
+    "entityId": "067c4008-9cbe-422e-b697-05442f73da2b",
+    "fileId": "067c4008-9cbe-422e-b697-05442f73da2b",
+    "IPFS-Add": "MY_HASH",
+    "Tag-1": "Val",
+    "Tag-2": "Val",
+    "Tag-3": "Val",
+    "Boost": "1.05"
+}
+```
+
+#### Applying Unique Custom MetaData During Bulk Workflows
+
+With some custom scripting and the `--metadata-file` parameter, the ArDrive CLI can be used to apply custom metadata to each file individually in a bulk workflow. For example, if you choose a numbered file naming pattern you can make use of a `for` loop:
+
+```shell
+for i in {1..5}
+do
+ardrive upload-file -F f0c58c11-430c-4383-8e54-4d864cc7e927 --local-path "../uploads/test-file-$i.txt" -w "/path/to/wallet.json" --metadata-file "../custom/metadata-$i.json" --dry-run > "file-result-$i.json"
+done
+```
+
 ## Other Utility Operations
 
 ### Monitoring Transactions

package.json

@@ -1,14 +1,14 @@
 {
 	"name": "ardrive-cli",
-	"version": "1.17.0",
+	"version": "1.18.0",
 	"description": "The ArDrive Command Line Interface (CLI is a Node.js application for terminal-based ArDrive workflows. It also offers utility operations for securely interacting with Arweave wallets and inspecting various Arweave blockchain conditions.",
 	"main": "./lib/index.js",
 	"bin": {
 		"ardrive": "./lib/index.js"
 	},
 	"types": "./lib/index.d.ts",
 	"dependencies": {
-		"ardrive-core-js": "1.15.1",
+		"ardrive-core-js": "1.16.0",
 		"arweave": "1.11.4",
 		"axios": "^0.21.1",
 		"bn.js": "^5.2.1",

src/CLICommand/parameters_helper.test.ts

@@ -25,7 +25,10 @@ import {
 	MaxDepthParameter,
 	AllParameter,
 	GatewayParameter,
-	DryRunParameter
+	DryRunParameter,
+	MetaDataFileParameter,
+	MetaDataGqlTagsParameter,
+	MetadataJsonParameter
 } from '../parameter_declarations';
 import '../parameter_declarations';
 import { CLIAction } from './action';
@@ -684,4 +687,69 @@ describe('ParametersHelper class', () => {
 			});
 		});
 	});
+
+	describe('getCustomMetaData method', () => {
+		it('returns the expected custom metadata with the --metadata-file parameter', async () => {
+			const cmd = declareCommandWithParams(program, [MetaDataFileParameter]);
+
+			CLICommand.parse(program, [
+				...baseArgv,
+				testCommandName,
+				'--metadata-file',
+				'tests/stub_files/custom_metadata.json'
+			]);
+			await cmd.action.then((options) => {
+				const parameters = new ParametersHelper(options);
+				const metaDataResult = parameters.getCustomMetaData();
+
+				expect(metaDataResult).to.deep.equal({
+					metaDataGqlTags: { tag: 'val' },
+					metaDataJson: { 'json field': true }
+				});
+			});
+		});
+
+		it('returns the expected custom metadata with the --metadata-gql-tags parameter', async () => {
+			const cmd = declareCommandWithParams(program, [MetaDataGqlTagsParameter]);
+
+			CLICommand.parse(program, [
+				...baseArgv,
+				testCommandName,
+				'--metadata-gql-tags',
+				'Tag-1',
+				'Val 1',
+				'Tag 2',
+				'Val 2'
+			]);
+
+			await cmd.action.then((options) => {
+				const parameters = new ParametersHelper(options);
+				const metaDataResult = parameters.getCustomMetaData();
+
+				expect(metaDataResult).to.deep.equal({
+					metaDataGqlTags: { 'Tag 2': 'Val 2', 'Tag-1': 'Val 1' }
+				});
+			});
+		});
+
+		it('returns the expected custom metadata with the --metadata-json parameter', async () => {
+			const cmd = declareCommandWithParams(program, [MetadataJsonParameter]);
+
+			CLICommand.parse(program, [
+				...baseArgv,
+				testCommandName,
+				'--metadata-json',
+				'{"key": "val", "key-2": true, "key-3": 420, "key-4": ["more", 1337]}'
+			]);
+
+			await cmd.action.then((options) => {
+				const parameters = new ParametersHelper(options);
+				const metaDataResult = parameters.getCustomMetaData();
+
+				expect(metaDataResult).to.deep.equal({
+					metaDataJson: { key: 'val', 'key-2': true, 'key-3': 420, 'key-4': ['more', 1337] }
+				});
+			});
+		});
+	});
 });

src/CLICommand/parameters_helper.ts

@@ -14,7 +14,10 @@ import {
 	SkipParameter,
 	BoostParameter,
 	GatewayParameter,
-	DryRunParameter
+	DryRunParameter,
+	MetaDataFileParameter,
+	MetaDataGqlTagsParameter,
+	MetadataJsonParameter
 } from '../parameter_declarations';
 import { cliWalletDao } from '..';
 import passwordPrompt from 'prompts';
@@ -35,7 +38,10 @@ import {
 	skipOnConflicts,
 	upsertOnConflicts,
 	askOnConflicts,
-	EntityKey
+	EntityKey,
+	CustomMetaData,
+	assertCustomMetaData,
+	CustomMetaDataJsonFields
 } from 'ardrive-core-js';
 import { JWKInterface } from 'arweave/node/lib/wallet';
 
@@ -282,6 +288,79 @@ export class ParametersHelper {
 		return mapFunc(value);
 	}
 
+	public getCustomMetaData(): CustomMetaData | undefined {
+		const metaDataPath = this.getParameterValue(MetaDataFileParameter);
+
+		const customMetaData = (() => {
+			if (metaDataPath) {
+				return this.readMetaDataFromPath(metaDataPath);
+			}
+
+			const customMetaData: CustomMetaData = {};
+			const metaDataGqlTags = this.mapMetaDataArrayToCustomMetaDataShape(
+				this.getParameterValue<string[]>(MetaDataGqlTagsParameter)
+			);
+			if (metaDataGqlTags) {
+				Object.assign(customMetaData, { metaDataGqlTags });
+			}
+
+			const metaDataJson = this.getParameterValue<string>(MetadataJsonParameter);
+
+			if (metaDataJson) {
+				Object.assign(customMetaData, { metaDataJson: this.parseMetaDataJson(metaDataJson) });
+			}
+			return customMetaData;
+		})();
+
+		if (Object.keys(customMetaData).length > 0) {
+			assertCustomMetaData(customMetaData);
+			return customMetaData;
+		}
+
+		return undefined;
+	}
+
+	private readMetaDataFromPath(path: string): CustomMetaData {
+		// Read tag file or throw fs path error or throw JSON parse error
+		const tagFile = fs.readFileSync(path, { encoding: 'utf8' });
+		return JSON.parse(tagFile);
+	}
+
+	private mapMetaDataArrayToCustomMetaDataShape(
+		metadata: string[] | undefined
+	): CustomMetaDataCliArrayInput | undefined {
+		if (metadata === undefined || metadata.length === 0) {
+			return undefined;
+		}
+		this.assertEvenTags(metadata);
+
+		const metaData: CustomMetaDataCliArrayInput = {};
+		let temp: string | null = null;
+
+		for (const val of metadata) {
+			if (temp === null) {
+				// val is tag Name
+				temp = val;
+			} else {
+				// val is tag Value
+				metaData[temp] = val;
+				temp = null;
+			}
+		}
+
+		return metaData;
+	}
+
+	private assertEvenTags(tags?: string[]): void {
+		if (tags && tags.length % 2 !== 0) {
+			throw Error('User must provide an even number custom metadata inputs! e.g: --metadata-json "NAME" "VALUE"');
+		}
+	}
+
+	private parseMetaDataJson(rawMetaDataJsonString: string): CustomMetaDataJsonFields {
+		return JSON.parse(rawMetaDataJsonString);
+	}
+
 	/**
 	 * Gathers a valid gateway URL from user provided gateway parameter,
 	 * an environment variable, or returns the default arweave gateway
@@ -323,3 +402,5 @@ export class ParametersHelper {
 		return !!dryRun;
 	}
 }
+
+type CustomMetaDataCliArrayInput = Record<string, string>;