From a8832f351224f9267ccb9e26c26d5d4b12d81927 Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Thu, 28 Oct 2021 13:27:43 -0500 Subject: [PATCH 01/31] feat(skip): Initialize conflict resolution params for single upload PE-638 --- src/ardrive.ts | 23 ++++++++++++++++++++--- src/commands/upload_file.ts | 29 +++++++++++++++++++++++++++-- src/index.ts | 9 ++++++--- src/parameter_declarations.ts | 22 ++++++++++++++++++++++ 4 files changed, 75 insertions(+), 8 deletions(-) diff --git a/src/ardrive.ts b/src/ardrive.ts index 876dea3e..d97b0ff9 100644 --- a/src/ardrive.ts +++ b/src/ardrive.ts @@ -179,6 +179,8 @@ export class ArDriveAnonymous extends ArDriveType { } } +export type ConflictResolution = 'skip' | 'replace'; // | 'upsert' | 'ask' + export class ArDrive extends ArDriveAnonymous { constructor( private readonly wallet: Wallet, @@ -189,7 +191,8 @@ export class ArDrive extends ArDriveAnonymous { private readonly appVersion: string, private readonly priceEstimator: ARDataPriceEstimator = new ARDataPriceRegressionEstimator(true), private readonly feeMultiple: FeeMultiple = 1.0, - private readonly dryRun: boolean = false + private readonly dryRun: boolean = false, + private readonly conflictResolution: ConflictResolution = 'skip' ) { super(arFsDao); } @@ -499,9 +502,16 @@ export class ArDrive extends ArDriveAnonymous { throw new Error(errorMessage.entityNameExists); } + const conflictingFileName = filesAndFolderNames.files.find((f) => f.fileName === destFileName); + + if (conflictingFileName && this.conflictResolution === 'skip') { + throw new Error(errorMessage.entityNameExists); + } + // TODO: Handle this.conflictResolution === 'upsert' and 'ask' PE-638 + // File is a new revision if destination name conflicts // with an existing file in the destination folder - const existingFileId = filesAndFolderNames.files.find((f) => f.fileName === destFileName)?.fileId; + const existingFileId = conflictingFileName?.fileId; const uploadBaseCosts = await this.estimateAndAssertCostOfFileUpload( wrappedFile.fileStats.size, @@ -750,9 +760,16 @@ export class ArDrive extends ArDriveAnonymous { throw new Error(errorMessage.entityNameExists); } + const conflictingFileName = filesAndFolderNames.files.find((f) => f.fileName === destFileName); + + if (conflictingFileName && this.conflictResolution === 'skip') { + throw new Error(errorMessage.entityNameExists); + } + // TODO: Handle this.conflictResolution === 'upsert' and 'ask' PE-638 + // File is a new revision if destination name conflicts // with an existing file in the destination folder - const existingFileId = filesAndFolderNames.files.find((f) => f.fileName === destFileName)?.fileId; + const existingFileId = conflictingFileName?.fileId; const uploadBaseCosts = await this.estimateAndAssertCostOfFileUpload( wrappedFile.fileStats.size, diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index 9d9ad263..ad8fae89 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -3,12 +3,15 @@ import { arDriveFactory } from '..'; import { CLICommand, ParametersHelper } from '../CLICommand'; import { BoostParameter, + ConflictResolutionParams, DestinationFileNameParameter, DrivePrivacyParameters, DryRunParameter, LocalFilePathParameter, LocalFilesParameter, - ParentFolderIdParameter + ReplaceParameter, + ParentFolderIdParameter, + SkipParameter } from '../parameter_declarations'; import { DriveKey, FeeMultiple, FolderID } from '../types'; import { readJWKFile } from '../utils'; @@ -31,6 +34,7 @@ new CLICommand({ LocalFilesParameter, BoostParameter, DryRunParameter, + ...ConflictResolutionParams, ...DrivePrivacyParameters ], async action(options) { @@ -74,10 +78,31 @@ new CLICommand({ const wallet = readJWKFile(options.walletFile); + const conflictResolution = (() => { + if (parameters.getParameterValue(SkipParameter)) { + return 'skip'; + } + + if (parameters.getParameterValue(ReplaceParameter)) { + return 'replace'; + } + + // if (parameters.getParameterValue(UpsertParameter)) { + // return 'upsert' + // }; + + // if (parameters.getParameterValue(AskParameter)) { + // return 'ask' + // }; + + return undefined; + })(); + const arDrive = arDriveFactory({ wallet: wallet, feeMultiple: options.boost as FeeMultiple, - dryRun: options.dryRun + dryRun: options.dryRun, + conflictResolution }); await Promise.all( diff --git a/src/index.ts b/src/index.ts index ea5b3d13..07222325 100755 --- a/src/index.ts +++ b/src/index.ts @@ -3,7 +3,7 @@ import { Wallet, WalletDAO } from './wallet'; import Arweave from 'arweave'; import { ArDriveCommunityOracle } from './community/ardrive_community_oracle'; -import { ArDrive, ArDriveAnonymous } from './ardrive'; +import { ArDrive, ArDriveAnonymous, ConflictResolution } from './ardrive'; import { ArFSDAO } from './arfsdao'; import { ARDataPriceEstimator } from './utils/ar_data_price_estimator'; import { ARDataPriceRegressionEstimator } from './utils/ar_data_price_regression_estimator'; @@ -47,6 +47,7 @@ export interface ArDriveSettings extends ArDriveSettingsAnonymous { feeMultiple?: FeeMultiple; dryRun?: boolean; arfsDao?: ArFSDAO; + conflictResolution?: ConflictResolution; } export function arDriveFactory({ @@ -57,7 +58,8 @@ export function arDriveFactory({ walletDao = cliWalletDao, dryRun, feeMultiple, - arfsDao = new ArFSDAO(wallet, arweave, dryRun, CLI_APP_NAME, CLI_APP_VERSION) + arfsDao = new ArFSDAO(wallet, arweave, dryRun, CLI_APP_NAME, CLI_APP_VERSION), + conflictResolution }: ArDriveSettings): ArDrive { return new ArDrive( wallet, @@ -68,7 +70,8 @@ export function arDriveFactory({ CLI_APP_VERSION, priceEstimator, feeMultiple, - dryRun + dryRun, + conflictResolution ); } diff --git a/src/parameter_declarations.ts b/src/parameter_declarations.ts index 971ca69b..0e9c2d17 100644 --- a/src/parameter_declarations.ts +++ b/src/parameter_declarations.ts @@ -24,6 +24,10 @@ export const AllParameter = 'all'; export const MaxDepthParameter = 'maxDepth'; export const BoostParameter = 'boost'; export const DryRunParameter = 'dryRun'; +export const SkipParameter = 'skip'; +export const ReplaceParameter = 'replace'; +// export const UpsertParameter = 'upsert'; +// export const AskParameter = 'ask'; export const NoVerifyParameter = 'verify'; // commander maps --no-x style params to options.x and always includes in options // Aggregates for convenience @@ -36,6 +40,8 @@ export const DriveCreationPrivacyParameters = [ export const DrivePrivacyParameters = [DriveKeyParameter, ...DriveCreationPrivacyParameters]; export const TreeDepthParams = [AllParameter, MaxDepthParameter]; +export const ConflictResolutionParams = [SkipParameter, ReplaceParameter /* , UpsertParameter, AskParameter */]; + /** * Note: importing this file will declare all the above parameters */ @@ -215,6 +221,22 @@ Parameter.declare({ type: 'boolean' }); +Parameter.declare({ + name: SkipParameter, + aliases: ['--skip'], + description: '(OPTIONAL) Skip upload if there is a name conflict within destination folder', + type: 'boolean', + forbiddenConjunctionParameters: [ReplaceParameter] +}); + +Parameter.declare({ + name: ReplaceParameter, + aliases: ['--replace'], + description: '(OPTIONAL) Create new file revisions if there is a name conflict within destination folder', + type: 'boolean', + forbiddenConjunctionParameters: [SkipParameter] +}); + Parameter.declare({ name: AllParameter, aliases: ['--all'], From f94702faa0e48c29bcecd4f5eadca7089e7d3751 Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Thu, 28 Oct 2021 13:38:50 -0500 Subject: [PATCH 02/31] test(skip): Add new conflict resolution cases to upload test PE-638 --- tests/integration/ardrive.int.test.ts | 48 ++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/tests/integration/ardrive.int.test.ts b/tests/integration/ardrive.int.test.ts index 7e5360a7..1f90cfd0 100644 --- a/tests/integration/ardrive.int.test.ts +++ b/tests/integration/ardrive.int.test.ts @@ -64,6 +64,19 @@ describe('ArDrive class - integrated', () => { true ); + const arDriveReplace = new ArDrive( + wallet, + walletDao, + arfsDao, + communityOracle, + 'Integration Test', + '1.0', + priceEstimator, + 1.0, + true, + 'replace' + ); + const walletOwner = stubArweaveAddress(); const unexpectedOwner = stubArweaveAddress('0987654321klmnopqrxtuvwxyz123456789ABCDEFGH'); @@ -482,10 +495,23 @@ describe('ArDrive class - integrated', () => { }); }); - it('returns the correct ArFSResult revision if destination folder has a conflicting FILE name', async () => { + it('throws an error if destination folder has a conflicting FILE name and conflict resolution is set to skip', async () => { stub(arfsDao, 'getOwnerForDriveId').resolves(walletOwner); - const result = await arDrive.uploadPublicFile(stubEntityID, wrappedFile, 'CONFLICTING_FILE_NAME'); + await expectAsyncErrorThrow({ + promiseToError: arDrive.uploadPublicFile(stubEntityID, wrappedFile, 'CONFLICTING_FILE_NAME'), + errorMessage: 'Entity name already exists in destination folder!' + }); + }); + + it('returns the correct ArFSResult revision if destination folder has a conflicting FILE name and conflict resolution is set to replace', async () => { + stub(arfsDao, 'getOwnerForDriveId').resolves(walletOwner); + + const result = await arDriveReplace.uploadPublicFile( + stubEntityID, + wrappedFile, + 'CONFLICTING_FILE_NAME' + ); // Pass expected existing file id, so that the file would be considered a revision assertUploadFileExpectations(result, 3204, 171, 0, '1', 'public', existingFileId); @@ -535,10 +561,24 @@ describe('ArDrive class - integrated', () => { }); }); - it('returns the correct ArFSResult revision if destination folder has a conflicting FILE name', async () => { + it('throws an error if destination folder has a conflicting FILE name and conflict resolution is set to skip', async () => { + stub(arfsDao, 'getOwnerForDriveId').resolves(walletOwner); + + await expectAsyncErrorThrow({ + promiseToError: arDrive.uploadPrivateFile( + stubEntityID, + wrappedFile, + await getStubDriveKey(), + 'CONFLICTING_FILE_NAME' + ), + errorMessage: 'Entity name already exists in destination folder!' + }); + }); + + it('returns the correct ArFSResult revision with if destination folder has a conflicting FILE name and conflict resolution is set to replace', async () => { stub(arfsDao, 'getOwnerForDriveId').resolves(walletOwner); - const result = await arDrive.uploadPrivateFile( + const result = await arDriveReplace.uploadPrivateFile( stubEntityID, wrappedFile, await getStubDriveKey(), From 7b7a16813c1c6095fc605992ca6c14a67a975ec0 Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Thu, 28 Oct 2021 13:45:31 -0500 Subject: [PATCH 03/31] feat(skip): Add skip logic to bulk upload PE-638 --- src/ardrive.ts | 142 ++++++++++++++++++++++++++----------------------- 1 file changed, 74 insertions(+), 68 deletions(-) diff --git a/src/ardrive.ts b/src/ardrive.ts index d97b0ff9..974f3544 100644 --- a/src/ardrive.ts +++ b/src/ardrive.ts @@ -667,40 +667,43 @@ export class ArDrive extends ArDriveAnonymous { // Upload all files in the folder for await (const wrappedFile of wrappedFolder.files) { - const fileDataRewardSettings = { - reward: wrappedFile.getBaseCosts().fileDataBaseReward, - feeMultiple: this.feeMultiple - }; - - const metadataRewardSettings = { - reward: wrappedFile.getBaseCosts().metaDataBaseReward, - feeMultiple: this.feeMultiple - }; - - const uploadFileResult = await this.arFsDao.uploadPublicFile({ - parentFolderId: folderId, - wrappedFile, - driveId, - fileDataRewardSettings, - metadataRewardSettings, - existingFileId: wrappedFile.existingId - }); + // Don't upload this file if there is an existing file and conflict resolution is skip + if (!(this.conflictResolution === 'skip' && wrappedFile.existingId)) { + const fileDataRewardSettings = { + reward: wrappedFile.getBaseCosts().fileDataBaseReward, + feeMultiple: this.feeMultiple + }; - // Capture all file results - uploadEntityFees = { - ...uploadEntityFees, - [uploadFileResult.dataTrxId]: +uploadFileResult.dataTrxReward, - [uploadFileResult.metaDataTrxId]: +uploadFileResult.metaDataTrxReward - }; - uploadEntityResults = [ - ...uploadEntityResults, - { - type: 'file', - metadataTxId: uploadFileResult.metaDataTrxId, - dataTxId: uploadFileResult.dataTrxId, - entityId: uploadFileResult.fileId - } - ]; + const metadataRewardSettings = { + reward: wrappedFile.getBaseCosts().metaDataBaseReward, + feeMultiple: this.feeMultiple + }; + + const uploadFileResult = await this.arFsDao.uploadPublicFile({ + parentFolderId: folderId, + wrappedFile, + driveId, + fileDataRewardSettings, + metadataRewardSettings, + existingFileId: wrappedFile.existingId + }); + + // Capture all file results + uploadEntityFees = { + ...uploadEntityFees, + [uploadFileResult.dataTrxId]: +uploadFileResult.dataTrxReward, + [uploadFileResult.metaDataTrxId]: +uploadFileResult.metaDataTrxReward + }; + uploadEntityResults = [ + ...uploadEntityResults, + { + type: 'file', + metadataTxId: uploadFileResult.metaDataTrxId, + dataTxId: uploadFileResult.dataTrxId, + entityId: uploadFileResult.fileId + } + ]; + } } // Upload folders, and children of those folders @@ -1020,41 +1023,44 @@ export class ArDrive extends ArDriveAnonymous { // Upload all files in the folder for await (const wrappedFile of wrappedFolder.files) { - const fileDataRewardSettings = { - reward: wrappedFile.getBaseCosts().fileDataBaseReward, - feeMultiple: this.feeMultiple - }; - const metadataRewardSettings = { - reward: wrappedFile.getBaseCosts().metaDataBaseReward, - feeMultiple: this.feeMultiple - }; - - const uploadFileResult = await this.arFsDao.uploadPrivateFile({ - parentFolderId: folderId, - wrappedFile, - driveId, - driveKey, - fileDataRewardSettings, - metadataRewardSettings, - existingFileId: wrappedFile.existingId - }); - - // Capture all file results - uploadEntityFees = { - ...uploadEntityFees, - [uploadFileResult.dataTrxId]: +uploadFileResult.dataTrxReward, - [uploadFileResult.metaDataTrxId]: +uploadFileResult.metaDataTrxReward - }; - uploadEntityResults = [ - ...uploadEntityResults, - { - type: 'file', - metadataTxId: uploadFileResult.metaDataTrxId, - dataTxId: uploadFileResult.dataTrxId, - entityId: uploadFileResult.fileId, - key: urlEncodeHashKey(uploadFileResult.fileKey) - } - ]; + // Don't upload this file if there is an existing file and conflict resolution is skip + if (!(this.conflictResolution === 'skip' && wrappedFile.existingId)) { + const fileDataRewardSettings = { + reward: wrappedFile.getBaseCosts().fileDataBaseReward, + feeMultiple: this.feeMultiple + }; + const metadataRewardSettings = { + reward: wrappedFile.getBaseCosts().metaDataBaseReward, + feeMultiple: this.feeMultiple + }; + + const uploadFileResult = await this.arFsDao.uploadPrivateFile({ + parentFolderId: folderId, + wrappedFile, + driveId, + driveKey, + fileDataRewardSettings, + metadataRewardSettings, + existingFileId: wrappedFile.existingId + }); + + // Capture all file results + uploadEntityFees = { + ...uploadEntityFees, + [uploadFileResult.dataTrxId]: +uploadFileResult.dataTrxReward, + [uploadFileResult.metaDataTrxId]: +uploadFileResult.metaDataTrxReward + }; + uploadEntityResults = [ + ...uploadEntityResults, + { + type: 'file', + metadataTxId: uploadFileResult.metaDataTrxId, + dataTxId: uploadFileResult.dataTrxId, + entityId: uploadFileResult.fileId, + key: urlEncodeHashKey(uploadFileResult.fileKey) + } + ]; + } } // Upload folders, and children of those folders From cb18fa9e7854d6cd3e3174bcaffabc0f7d0b40c6 Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Thu, 28 Oct 2021 13:51:32 -0500 Subject: [PATCH 04/31] docs(skip): Add skip logic README PE-638 --- README.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index aab4ad88..e6e60df5 100644 --- a/README.md +++ b/README.md @@ -690,6 +690,12 @@ NOTE: To upload to the root of a drive, specify its root folder ID as the parent ardrive drive-info -d "c7f87712-b54e-4491-bc96-1c5fa7b1da50" | jq -r '.rootFolderId' ``` +By default, the single `upload-file` command will skip name conflicts found. To override this behavior and make a new revision of a file, use the `--replace` option: + +```shell +ardrive upload-file --replace --local-file-path /path/to/file.txt --parent-folder-id "9af694f6-4cfc-4eee-88a8-1b02704760c0" -w /path/to/wallet.json +``` + ### Uploading a Folder with Files (Bulk Upload) Users can perform a bulk upload by using the upload-file command on a target folder. The command will reconstruct the folder hierarchy on local disk as ArFS folders on the permaweb and upload each file into their corresponding folders: @@ -698,12 +704,14 @@ Users can perform a bulk upload by using the upload-file command on a target fol ardrive upload-file --local-file-path /path/to/folder --parent-folder-id "9af694f6-4cfc-4eee-88a8-1b02704760c0" -w /path/to/wallet.json ``` -This method of upload can be used to upload a large number of files and folders within the folder tree. If existing entities are encountered in the destination folder tree that would cause naming conflicts, expect the following behaviors: +This method of upload can be used to upload a large number of files and folders within the folder tree. If existing entities are encountered in the destination folder tree that would cause naming conflicts, expect the following default behaviors: - Folder names that conflict with a FILE name at the destination will cause an error to be thrown - Folder names that conflict with a FOLDER name at the destination will use the existing folder ID (i.e. skip) rather than creating a new folder - File names that conflict with a FOLDER name at the destination will cause an error to be thrown -- File names that conflict with a FILE name at the destination will be uploaded as a REVISION +- File names that conflict with a FILE name at the destination will be SKIPPED + +Similar to the single file upload, the above FILE to FILE name conflict resolution behavior can be modified by the `--replace` command. This will force new revisions on all conflicts within the bulk upload. ### Fetching the Metadata of a File Entity From 73434d16d3662bcfbf4486d8a2450a67ef047a59 Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Thu, 28 Oct 2021 13:59:43 -0500 Subject: [PATCH 05/31] refactor(skip): Get conflict resolution params in helper to later share PE-638 --- src/CLICommand/parameters_helper.ts | 25 ++++++++++++++++++++++++- src/commands/upload_file.ts | 24 ++---------------------- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/src/CLICommand/parameters_helper.ts b/src/CLICommand/parameters_helper.ts index e65a4506..6d2ab6ca 100644 --- a/src/CLICommand/parameters_helper.ts +++ b/src/CLICommand/parameters_helper.ts @@ -10,13 +10,16 @@ import { MaxDepthParameter, SeedPhraseParameter, WalletFileParameter, - PrivateParameter + PrivateParameter, + ReplaceParameter, + SkipParameter } from '../parameter_declarations'; import { cliWalletDao } from '..'; import { DriveID, DriveKey } from '../types'; import passwordPrompt from 'prompts'; import { PrivateKeyData } from '../private_key_data'; import { ArweaveAddress } from '../arweave_address'; +import { ConflictResolution } from '../ardrive'; // eslint-disable-next-line @typescript-eslint/no-explicit-any type ParameterOptions = any; @@ -204,6 +207,26 @@ export class ParametersHelper { return maxDepthValue; } + public getConflictResolution(): ConflictResolution | undefined { + if (this.getParameterValue(SkipParameter)) { + return 'skip'; + } + + if (this.getParameterValue(ReplaceParameter)) { + return 'replace'; + } + + // if (this.getParameterValue(UpsertParameter)) { + // return 'upsert' + // }; + + // if (this.getParameterValue(AskParameter)) { + // return 'ask' + // }; + + return undefined; + } + /** * @param {ParameterName} parameterName * @returns {string | undefined} diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index ad8fae89..ad397cea 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -9,9 +9,7 @@ import { DryRunParameter, LocalFilePathParameter, LocalFilesParameter, - ReplaceParameter, - ParentFolderIdParameter, - SkipParameter + ParentFolderIdParameter } from '../parameter_declarations'; import { DriveKey, FeeMultiple, FolderID } from '../types'; import { readJWKFile } from '../utils'; @@ -78,25 +76,7 @@ new CLICommand({ const wallet = readJWKFile(options.walletFile); - const conflictResolution = (() => { - if (parameters.getParameterValue(SkipParameter)) { - return 'skip'; - } - - if (parameters.getParameterValue(ReplaceParameter)) { - return 'replace'; - } - - // if (parameters.getParameterValue(UpsertParameter)) { - // return 'upsert' - // }; - - // if (parameters.getParameterValue(AskParameter)) { - // return 'ask' - // }; - - return undefined; - })(); + const conflictResolution = parameters.getConflictResolution(); const arDrive = arDriveFactory({ wallet: wallet, From ebddd73c2860df79516bd0b4b50cac26ae7058dd Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Thu, 28 Oct 2021 20:40:42 -0500 Subject: [PATCH 06/31] refactor(skip): Handle conflict resolution as upload file param PE-638 --- src/CLICommand/parameters_helper.ts | 13 ++-- src/ardrive.ts | 107 +++++++++++++++++----------- src/commands/upload_file.ts | 39 +++++----- src/index.ts | 9 +-- 4 files changed, 97 insertions(+), 71 deletions(-) diff --git a/src/CLICommand/parameters_helper.ts b/src/CLICommand/parameters_helper.ts index 6d2ab6ca..65ee67b1 100644 --- a/src/CLICommand/parameters_helper.ts +++ b/src/CLICommand/parameters_helper.ts @@ -11,15 +11,14 @@ import { SeedPhraseParameter, WalletFileParameter, PrivateParameter, - ReplaceParameter, - SkipParameter + ReplaceParameter } from '../parameter_declarations'; import { cliWalletDao } from '..'; import { DriveID, DriveKey } from '../types'; import passwordPrompt from 'prompts'; import { PrivateKeyData } from '../private_key_data'; import { ArweaveAddress } from '../arweave_address'; -import { ConflictResolution } from '../ardrive'; +import { FileNameConflictResolution } from '../ardrive'; // eslint-disable-next-line @typescript-eslint/no-explicit-any type ParameterOptions = any; @@ -207,11 +206,7 @@ export class ParametersHelper { return maxDepthValue; } - public getConflictResolution(): ConflictResolution | undefined { - if (this.getParameterValue(SkipParameter)) { - return 'skip'; - } - + public getFileNameConflictResolution(): FileNameConflictResolution { if (this.getParameterValue(ReplaceParameter)) { return 'replace'; } @@ -224,7 +219,7 @@ export class ParametersHelper { // return 'ask' // }; - return undefined; + return 'skip'; } /** diff --git a/src/ardrive.ts b/src/ardrive.ts index 974f3544..15781914 100644 --- a/src/ardrive.ts +++ b/src/ardrive.ts @@ -102,6 +102,7 @@ interface RecursivePublicBulkUploadParams { wrappedFolder: ArFSFolderToUpload; driveId: DriveID; owner: ArweaveAddress; + conflictResolution: FileNameConflictResolution; } type RecursivePrivateBulkUploadParams = RecursivePublicBulkUploadParams & WithDriveKey; @@ -118,6 +119,25 @@ interface MovePublicFolderParams { } type MovePrivateFolderParams = MovePublicFolderParams & WithDriveKey; +export type FileNameConflictResolution = 'skip' | 'replace'; // | 'upsert' | 'ask' + +export interface UploadParams { + parentFolderId: FolderID; + conflictResolution?: FileNameConflictResolution; +} + +export interface BulkPublicUploadParams extends UploadParams { + wrappedFolder: ArFSFolderToUpload; + destParentFolderName?: string; +} +export type BulkPrivateUploadParams = BulkPublicUploadParams & WithDriveKey; + +export interface UploadPublicFileParams extends UploadParams { + wrappedFile: ArFSFileToUpload; + destinationFileName?: string; +} +export type UploadPrivateFileParams = UploadPublicFileParams & WithDriveKey; + export abstract class ArDriveType { protected abstract readonly arFsDao: ArFSDAOType; } @@ -179,8 +199,6 @@ export class ArDriveAnonymous extends ArDriveType { } } -export type ConflictResolution = 'skip' | 'replace'; // | 'upsert' | 'ask' - export class ArDrive extends ArDriveAnonymous { constructor( private readonly wallet: Wallet, @@ -191,8 +209,7 @@ export class ArDrive extends ArDriveAnonymous { private readonly appVersion: string, private readonly priceEstimator: ARDataPriceEstimator = new ARDataPriceRegressionEstimator(true), private readonly feeMultiple: FeeMultiple = 1.0, - private readonly dryRun: boolean = false, - private readonly conflictResolution: ConflictResolution = 'skip' + private readonly dryRun: boolean = false ) { super(arFsDao); } @@ -482,11 +499,12 @@ export class ArDrive extends ArDriveAnonymous { }); } - async uploadPublicFile( - parentFolderId: FolderID, - wrappedFile: ArFSFileToUpload, - destinationFileName?: string - ): Promise { + async uploadPublicFile({ + parentFolderId, + wrappedFile, + destinationFileName, + conflictResolution = 'skip' + }: UploadPublicFileParams): Promise { const driveId = await this.arFsDao.getDriveIdForFolderId(parentFolderId); const owner = await this.getOwnerForDriveId(driveId); @@ -504,7 +522,7 @@ export class ArDrive extends ArDriveAnonymous { const conflictingFileName = filesAndFolderNames.files.find((f) => f.fileName === destFileName); - if (conflictingFileName && this.conflictResolution === 'skip') { + if (conflictingFileName && conflictResolution === 'skip') { throw new Error(errorMessage.entityNameExists); } // TODO: Handle this.conflictResolution === 'upsert' and 'ask' PE-638 @@ -553,18 +571,19 @@ export class ArDrive extends ArDriveAnonymous { }); } - public async createPublicFolderAndUploadChildren( - parentFolderId: FolderID, - wrappedFolder: ArFSFolderToUpload, - parentFolderName?: string - ): Promise { + public async createPublicFolderAndUploadChildren({ + parentFolderId, + wrappedFolder, + destParentFolderName, + conflictResolution = 'skip' + }: BulkPublicUploadParams): Promise { const driveId = await this.arFsDao.getDriveIdForFolderId(parentFolderId); const owner = await this.getOwnerForDriveId(driveId); await this.assertOwnerAddress(owner); // Derive destination name and names already within provided destination folder - const destFolderName = parentFolderName ?? wrappedFolder.getBaseFileName(); + const destFolderName = destParentFolderName ?? wrappedFolder.getBaseFileName(); const filesAndFolderNames = await this.arFsDao.getPublicEntityNamesAndIdsInFolder(parentFolderId); // Folders cannot overwrite file names @@ -576,7 +595,7 @@ export class ArDrive extends ArDriveAnonymous { // Use existing folder id if the intended destination name // conflicts with an existing folder in the destination folder wrappedFolder.existingId = filesAndFolderNames.folders.find((f) => f.folderName === destFolderName)?.folderId; - wrappedFolder.destinationName = parentFolderName; + wrappedFolder.destinationName = destParentFolderName; // Check for conflicting names and assign existing IDs for later use await this.checkAndAssignExistingPublicNames(wrappedFolder); @@ -591,7 +610,8 @@ export class ArDrive extends ArDriveAnonymous { parentFolderId, wrappedFolder, driveId, - owner: await this.wallet.getAddress() + owner: await this.wallet.getAddress(), + conflictResolution }); if (+bulkEstimation.communityWinstonTip > 0) { @@ -620,7 +640,8 @@ export class ArDrive extends ArDriveAnonymous { parentFolderId, wrappedFolder, driveId, - owner + owner, + conflictResolution }: RecursivePublicBulkUploadParams): Promise<{ entityResults: ArFSEntityData[]; feeResults: ArFSFees; @@ -668,7 +689,7 @@ export class ArDrive extends ArDriveAnonymous { // Upload all files in the folder for await (const wrappedFile of wrappedFolder.files) { // Don't upload this file if there is an existing file and conflict resolution is skip - if (!(this.conflictResolution === 'skip' && wrappedFile.existingId)) { + if (!(conflictResolution === 'skip' && wrappedFile.existingId)) { const fileDataRewardSettings = { reward: wrappedFile.getBaseCosts().fileDataBaseReward, feeMultiple: this.feeMultiple @@ -713,7 +734,8 @@ export class ArDrive extends ArDriveAnonymous { parentFolderId: folderId, wrappedFolder: childFolder, driveId, - owner + owner, + conflictResolution }); // Capture all folder results @@ -742,12 +764,13 @@ export class ArDrive extends ArDriveAnonymous { return dataSize - modulo16 + 16; } - async uploadPrivateFile( - parentFolderId: FolderID, - wrappedFile: ArFSFileToUpload, - driveKey: DriveKey, - destinationFileName?: string - ): Promise { + async uploadPrivateFile({ + parentFolderId, + wrappedFile, + driveKey, + destinationFileName, + conflictResolution + }: UploadPrivateFileParams): Promise { const driveId = await this.arFsDao.getDriveIdForFolderId(parentFolderId); const owner = await this.getOwnerForDriveId(driveId); @@ -765,7 +788,7 @@ export class ArDrive extends ArDriveAnonymous { const conflictingFileName = filesAndFolderNames.files.find((f) => f.fileName === destFileName); - if (conflictingFileName && this.conflictResolution === 'skip') { + if (conflictingFileName && conflictResolution === 'skip') { throw new Error(errorMessage.entityNameExists); } // TODO: Handle this.conflictResolution === 'upsert' and 'ask' PE-638 @@ -825,12 +848,13 @@ export class ArDrive extends ArDriveAnonymous { }); } - public async createPrivateFolderAndUploadChildren( - parentFolderId: FolderID, - wrappedFolder: ArFSFolderToUpload, - driveKey: DriveKey, - parentFolderName?: string - ): Promise { + public async createPrivateFolderAndUploadChildren({ + parentFolderId, + wrappedFolder, + driveKey, + destParentFolderName, + conflictResolution = 'skip' + }: BulkPrivateUploadParams): Promise { // Retrieve drive ID from folder ID const driveId = await this.arFsDao.getDriveIdForFolderId(parentFolderId); @@ -841,7 +865,7 @@ export class ArDrive extends ArDriveAnonymous { await this.assertOwnerAddress(owner); // Derive destination name and names already within provided destination folder - const destFolderName = parentFolderName ?? wrappedFolder.getBaseFileName(); + const destFolderName = destParentFolderName ?? wrappedFolder.getBaseFileName(); const filesAndFolderNames = await this.arFsDao.getPrivateEntityNamesAndIdsInFolder(parentFolderId, driveKey); // Folders cannot overwrite file names @@ -853,7 +877,7 @@ export class ArDrive extends ArDriveAnonymous { // Use existing folder id if the intended destination name // conflicts with an existing folder in the destination folder wrappedFolder.existingId = filesAndFolderNames.folders.find((f) => f.folderName === destFolderName)?.folderId; - wrappedFolder.destinationName = parentFolderName; + wrappedFolder.destinationName = destParentFolderName; // Check for conflicting names and assign existing IDs for later use await this.checkAndAssignExistingPrivateNames(wrappedFolder, driveKey); @@ -869,7 +893,8 @@ export class ArDrive extends ArDriveAnonymous { wrappedFolder, driveKey, driveId, - owner + owner, + conflictResolution }); if (+bulkEstimation.communityWinstonTip > 0) { @@ -973,7 +998,8 @@ export class ArDrive extends ArDriveAnonymous { driveId, parentFolderId, driveKey, - owner + owner, + conflictResolution }: RecursivePrivateBulkUploadParams): Promise<{ entityResults: ArFSEntityData[]; feeResults: ArFSFees; @@ -1024,7 +1050,7 @@ export class ArDrive extends ArDriveAnonymous { // Upload all files in the folder for await (const wrappedFile of wrappedFolder.files) { // Don't upload this file if there is an existing file and conflict resolution is skip - if (!(this.conflictResolution === 'skip' && wrappedFile.existingId)) { + if (!(conflictResolution === 'skip' && wrappedFile.existingId)) { const fileDataRewardSettings = { reward: wrappedFile.getBaseCosts().fileDataBaseReward, feeMultiple: this.feeMultiple @@ -1071,7 +1097,8 @@ export class ArDrive extends ArDriveAnonymous { wrappedFolder: childFolder, driveId, driveKey, - owner + owner, + conflictResolution }); // Capture all folder results diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index ad397cea..25b56f82 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -76,13 +76,12 @@ new CLICommand({ const wallet = readJWKFile(options.walletFile); - const conflictResolution = parameters.getConflictResolution(); + const conflictResolution = parameters.getFileNameConflictResolution(); const arDrive = arDriveFactory({ wallet: wallet, feeMultiple: options.boost as FeeMultiple, - dryRun: options.dryRun, - conflictResolution + dryRun: options.dryRun }); await Promise.all( @@ -103,29 +102,37 @@ new CLICommand({ (await parameters.getDriveKey({ driveId, drivePassword, useCache: true })); if (isFolder(wrappedEntity)) { - return arDrive.createPrivateFolderAndUploadChildren( + return arDrive.createPrivateFolderAndUploadChildren({ parentFolderId, - wrappedEntity, + wrappedFolder: wrappedEntity, driveKey, - destinationFileName - ); + destParentFolderName: destinationFileName, + conflictResolution + }); } else { - return arDrive.uploadPrivateFile( + return arDrive.uploadPrivateFile({ parentFolderId, - wrappedEntity, + wrappedFile: wrappedEntity, driveKey, - destinationFileName - ); + destinationFileName, + conflictResolution + }); } } else { if (isFolder(wrappedEntity)) { - return arDrive.createPublicFolderAndUploadChildren( + return arDrive.createPublicFolderAndUploadChildren({ parentFolderId, - wrappedEntity, - destinationFileName - ); + wrappedFolder: wrappedEntity, + destParentFolderName: destinationFileName, + conflictResolution + }); } else { - return arDrive.uploadPublicFile(parentFolderId, wrappedEntity, destinationFileName); + return arDrive.uploadPublicFile({ + parentFolderId, + wrappedFile: wrappedEntity, + destinationFileName, + conflictResolution + }); } } })(); diff --git a/src/index.ts b/src/index.ts index 07222325..ea5b3d13 100755 --- a/src/index.ts +++ b/src/index.ts @@ -3,7 +3,7 @@ import { Wallet, WalletDAO } from './wallet'; import Arweave from 'arweave'; import { ArDriveCommunityOracle } from './community/ardrive_community_oracle'; -import { ArDrive, ArDriveAnonymous, ConflictResolution } from './ardrive'; +import { ArDrive, ArDriveAnonymous } from './ardrive'; import { ArFSDAO } from './arfsdao'; import { ARDataPriceEstimator } from './utils/ar_data_price_estimator'; import { ARDataPriceRegressionEstimator } from './utils/ar_data_price_regression_estimator'; @@ -47,7 +47,6 @@ export interface ArDriveSettings extends ArDriveSettingsAnonymous { feeMultiple?: FeeMultiple; dryRun?: boolean; arfsDao?: ArFSDAO; - conflictResolution?: ConflictResolution; } export function arDriveFactory({ @@ -58,8 +57,7 @@ export function arDriveFactory({ walletDao = cliWalletDao, dryRun, feeMultiple, - arfsDao = new ArFSDAO(wallet, arweave, dryRun, CLI_APP_NAME, CLI_APP_VERSION), - conflictResolution + arfsDao = new ArFSDAO(wallet, arweave, dryRun, CLI_APP_NAME, CLI_APP_VERSION) }: ArDriveSettings): ArDrive { return new ArDrive( wallet, @@ -70,8 +68,7 @@ export function arDriveFactory({ CLI_APP_VERSION, priceEstimator, feeMultiple, - dryRun, - conflictResolution + dryRun ); } From ea54df9c483f6919891011428c549382a168fe9b Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Thu, 28 Oct 2021 20:55:52 -0500 Subject: [PATCH 07/31] test(skip): Adjust tests for refactor PE-638 --- tests/integration/ardrive.int.test.ts | 83 +++++++++++++++------------ 1 file changed, 45 insertions(+), 38 deletions(-) diff --git a/tests/integration/ardrive.int.test.ts b/tests/integration/ardrive.int.test.ts index 1f90cfd0..1ad1772d 100644 --- a/tests/integration/ardrive.int.test.ts +++ b/tests/integration/ardrive.int.test.ts @@ -64,19 +64,6 @@ describe('ArDrive class - integrated', () => { true ); - const arDriveReplace = new ArDrive( - wallet, - walletDao, - arfsDao, - communityOracle, - 'Integration Test', - '1.0', - priceEstimator, - 1.0, - true, - 'replace' - ); - const walletOwner = stubArweaveAddress(); const unexpectedOwner = stubArweaveAddress('0987654321klmnopqrxtuvwxyz123456789ABCDEFGH'); @@ -481,7 +468,7 @@ describe('ArDrive class - integrated', () => { stub(arfsDao, 'getOwnerForDriveId').resolves(unexpectedOwner); await expectAsyncErrorThrow({ - promiseToError: arDrive.uploadPublicFile(stubEntityID, wrappedFile), + promiseToError: arDrive.uploadPublicFile({ parentFolderId: stubEntityID, wrappedFile }), errorMessage: 'Supplied wallet is not the owner of this drive!' }); }); @@ -490,7 +477,11 @@ describe('ArDrive class - integrated', () => { stub(arfsDao, 'getOwnerForDriveId').resolves(walletOwner); await expectAsyncErrorThrow({ - promiseToError: arDrive.uploadPublicFile(stubEntityID, wrappedFile, 'CONFLICTING_FOLDER_NAME'), + promiseToError: arDrive.uploadPublicFile({ + parentFolderId: stubEntityID, + wrappedFile, + destinationFileName: 'CONFLICTING_FOLDER_NAME' + }), errorMessage: 'Entity name already exists in destination folder!' }); }); @@ -499,7 +490,12 @@ describe('ArDrive class - integrated', () => { stub(arfsDao, 'getOwnerForDriveId').resolves(walletOwner); await expectAsyncErrorThrow({ - promiseToError: arDrive.uploadPublicFile(stubEntityID, wrappedFile, 'CONFLICTING_FILE_NAME'), + promiseToError: arDrive.uploadPublicFile({ + parentFolderId: stubEntityID, + wrappedFile, + destinationFileName: 'CONFLICTING_FILE_NAME', + conflictResolution: 'skip' + }), errorMessage: 'Entity name already exists in destination folder!' }); }); @@ -507,11 +503,12 @@ describe('ArDrive class - integrated', () => { it('returns the correct ArFSResult revision if destination folder has a conflicting FILE name and conflict resolution is set to replace', async () => { stub(arfsDao, 'getOwnerForDriveId').resolves(walletOwner); - const result = await arDriveReplace.uploadPublicFile( - stubEntityID, + const result = await arDrive.uploadPublicFile({ + parentFolderId: stubEntityID, wrappedFile, - 'CONFLICTING_FILE_NAME' - ); + destinationFileName: 'CONFLICTING_FILE_NAME', + conflictResolution: 'replace' + }); // Pass expected existing file id, so that the file would be considered a revision assertUploadFileExpectations(result, 3204, 171, 0, '1', 'public', existingFileId); @@ -520,7 +517,7 @@ describe('ArDrive class - integrated', () => { it('returns the correct ArFSResult', async () => { stub(arfsDao, 'getOwnerForDriveId').resolves(walletOwner); - const result = await arDrive.uploadPublicFile(stubEntityID, wrappedFile); + const result = await arDrive.uploadPublicFile({ parentFolderId: stubEntityID, wrappedFile }); assertUploadFileExpectations(result, 3204, 166, 0, '1', 'public'); }); }); @@ -542,7 +539,11 @@ describe('ArDrive class - integrated', () => { stub(arfsDao, 'getOwnerForDriveId').resolves(unexpectedOwner); await expectAsyncErrorThrow({ - promiseToError: arDrive.uploadPrivateFile(stubEntityID, wrappedFile, await getStubDriveKey()), + promiseToError: arDrive.uploadPrivateFile({ + parentFolderId: stubEntityID, + wrappedFile, + driveKey: await getStubDriveKey() + }), errorMessage: 'Supplied wallet is not the owner of this drive!' }); }); @@ -551,12 +552,12 @@ describe('ArDrive class - integrated', () => { stub(arfsDao, 'getOwnerForDriveId').resolves(walletOwner); await expectAsyncErrorThrow({ - promiseToError: arDrive.uploadPrivateFile( - stubEntityID, + promiseToError: arDrive.uploadPrivateFile({ + parentFolderId: stubEntityID, wrappedFile, - await getStubDriveKey(), - 'CONFLICTING_FOLDER_NAME' - ), + driveKey: await getStubDriveKey(), + destinationFileName: 'CONFLICTING_FOLDER_NAME' + }), errorMessage: 'Entity name already exists in destination folder!' }); }); @@ -565,12 +566,13 @@ describe('ArDrive class - integrated', () => { stub(arfsDao, 'getOwnerForDriveId').resolves(walletOwner); await expectAsyncErrorThrow({ - promiseToError: arDrive.uploadPrivateFile( - stubEntityID, + promiseToError: arDrive.uploadPrivateFile({ + parentFolderId: stubEntityID, wrappedFile, - await getStubDriveKey(), - 'CONFLICTING_FILE_NAME' - ), + driveKey: await getStubDriveKey(), + destinationFileName: 'CONFLICTING_FILE_NAME', + conflictResolution: 'skip' + }), errorMessage: 'Entity name already exists in destination folder!' }); }); @@ -578,12 +580,13 @@ describe('ArDrive class - integrated', () => { it('returns the correct ArFSResult revision with if destination folder has a conflicting FILE name and conflict resolution is set to replace', async () => { stub(arfsDao, 'getOwnerForDriveId').resolves(walletOwner); - const result = await arDriveReplace.uploadPrivateFile( - stubEntityID, + const result = await arDrive.uploadPrivateFile({ + parentFolderId: stubEntityID, wrappedFile, - await getStubDriveKey(), - 'CONFLICTING_FILE_NAME' - ); + driveKey: await getStubDriveKey(), + destinationFileName: 'CONFLICTING_FILE_NAME', + conflictResolution: 'replace' + }); // Pass expected existing file id, so that the file would be considered a revision assertUploadFileExpectations(result, 3216, 187, 0, '1', 'private', existingFileId); @@ -593,7 +596,11 @@ describe('ArDrive class - integrated', () => { stub(arfsDao, 'getOwnerForDriveId').resolves(walletOwner); const stubDriveKey = await getStubDriveKey(); - const result = await arDrive.uploadPrivateFile(stubEntityID, wrappedFile, stubDriveKey); + const result = await arDrive.uploadPrivateFile({ + parentFolderId: stubEntityID, + wrappedFile, + driveKey: stubDriveKey + }); assertUploadFileExpectations(result, 3216, 182, 0, '1', 'private'); }); }); From 82624b6b658c11e146d134502667a3cb7b8b5280 Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Thu, 28 Oct 2021 21:11:22 -0500 Subject: [PATCH 08/31] refactor(skip): Return empty ArFS result on skip PE-638 --- src/ardrive.ts | 10 ++++-- tests/integration/ardrive.int.test.ts | 44 +++++++++++++++------------ 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/src/ardrive.ts b/src/ardrive.ts index 15781914..efc5d37c 100644 --- a/src/ardrive.ts +++ b/src/ardrive.ts @@ -81,6 +81,12 @@ export interface ArFSResult { fees: ArFSFees; } +const emptyArFSResult: ArFSResult = { + created: [], + tips: [], + fees: {} +}; + export interface MetaDataBaseCosts { metaDataBaseReward: Winston; } @@ -523,7 +529,7 @@ export class ArDrive extends ArDriveAnonymous { const conflictingFileName = filesAndFolderNames.files.find((f) => f.fileName === destFileName); if (conflictingFileName && conflictResolution === 'skip') { - throw new Error(errorMessage.entityNameExists); + return emptyArFSResult; } // TODO: Handle this.conflictResolution === 'upsert' and 'ask' PE-638 @@ -789,7 +795,7 @@ export class ArDrive extends ArDriveAnonymous { const conflictingFileName = filesAndFolderNames.files.find((f) => f.fileName === destFileName); if (conflictingFileName && conflictResolution === 'skip') { - throw new Error(errorMessage.entityNameExists); + return emptyArFSResult; } // TODO: Handle this.conflictResolution === 'upsert' and 'ask' PE-638 diff --git a/tests/integration/ardrive.int.test.ts b/tests/integration/ardrive.int.test.ts index 1ad1772d..fa974975 100644 --- a/tests/integration/ardrive.int.test.ts +++ b/tests/integration/ardrive.int.test.ts @@ -486,17 +486,20 @@ describe('ArDrive class - integrated', () => { }); }); - it('throws an error if destination folder has a conflicting FILE name and conflict resolution is set to skip', async () => { + it('returns the correct empty ArFSResult if destination folder has a conflicting FILE name and conflict resolution is set to skip', async () => { stub(arfsDao, 'getOwnerForDriveId').resolves(walletOwner); - await expectAsyncErrorThrow({ - promiseToError: arDrive.uploadPublicFile({ - parentFolderId: stubEntityID, - wrappedFile, - destinationFileName: 'CONFLICTING_FILE_NAME', - conflictResolution: 'skip' - }), - errorMessage: 'Entity name already exists in destination folder!' + const result = await arDrive.uploadPublicFile({ + parentFolderId: stubEntityID, + wrappedFile, + destinationFileName: 'CONFLICTING_FILE_NAME', + conflictResolution: 'skip' + }); + + expect(result).to.deep.equal({ + created: [], + tips: [], + fees: {} }); }); @@ -562,18 +565,21 @@ describe('ArDrive class - integrated', () => { }); }); - it('throws an error if destination folder has a conflicting FILE name and conflict resolution is set to skip', async () => { + it('returns the correct empty ArFSResult if destination folder has a conflicting FILE name and conflict resolution is set to skip', async () => { stub(arfsDao, 'getOwnerForDriveId').resolves(walletOwner); - await expectAsyncErrorThrow({ - promiseToError: arDrive.uploadPrivateFile({ - parentFolderId: stubEntityID, - wrappedFile, - driveKey: await getStubDriveKey(), - destinationFileName: 'CONFLICTING_FILE_NAME', - conflictResolution: 'skip' - }), - errorMessage: 'Entity name already exists in destination folder!' + const result = await arDrive.uploadPrivateFile({ + parentFolderId: stubEntityID, + wrappedFile, + driveKey: await getStubDriveKey(), + destinationFileName: 'CONFLICTING_FILE_NAME', + conflictResolution: 'skip' + }); + + expect(result).to.deep.equal({ + created: [], + tips: [], + fees: {} }); }); From 02c120828d831c35ecbd7bfa446d3e80f322d101 Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Fri, 29 Oct 2021 20:07:24 -0500 Subject: [PATCH 09/31] feat(upsert): Initialize upsert param on uploads PE-633 --- src/CLICommand/parameters_helper.ts | 11 +- src/ardrive.ts | 210 +++++++++++++++++----------- src/arfs_file_wrapper.ts | 7 +- src/arfsdao.ts | 11 +- src/parameter_declarations.ts | 15 +- src/utils/mapper_functions.ts | 11 +- 6 files changed, 164 insertions(+), 101 deletions(-) diff --git a/src/CLICommand/parameters_helper.ts b/src/CLICommand/parameters_helper.ts index 65ee67b1..c6ac84b9 100644 --- a/src/CLICommand/parameters_helper.ts +++ b/src/CLICommand/parameters_helper.ts @@ -11,7 +11,8 @@ import { SeedPhraseParameter, WalletFileParameter, PrivateParameter, - ReplaceParameter + ReplaceParameter, + UpsertParameter } from '../parameter_declarations'; import { cliWalletDao } from '..'; import { DriveID, DriveKey } from '../types'; @@ -211,15 +212,15 @@ export class ParametersHelper { return 'replace'; } - // if (this.getParameterValue(UpsertParameter)) { - // return 'upsert' - // }; + if (this.getParameterValue(UpsertParameter)) { + return 'skip'; + } // if (this.getParameterValue(AskParameter)) { // return 'ask' // }; - return 'skip'; + return 'upsert'; } /** diff --git a/src/ardrive.ts b/src/ardrive.ts index efc5d37c..7de28902 100644 --- a/src/ardrive.ts +++ b/src/ardrive.ts @@ -125,7 +125,7 @@ interface MovePublicFolderParams { } type MovePrivateFolderParams = MovePublicFolderParams & WithDriveKey; -export type FileNameConflictResolution = 'skip' | 'replace'; // | 'upsert' | 'ask' +export type FileNameConflictResolution = 'skip' | 'replace' | 'upsert'; // | 'ask' export interface UploadParams { parentFolderId: FolderID; @@ -509,7 +509,7 @@ export class ArDrive extends ArDriveAnonymous { parentFolderId, wrappedFile, destinationFileName, - conflictResolution = 'skip' + conflictResolution = 'upsert' }: UploadPublicFileParams): Promise { const driveId = await this.arFsDao.getDriveIdForFolderId(parentFolderId); @@ -528,10 +528,22 @@ export class ArDrive extends ArDriveAnonymous { const conflictingFileName = filesAndFolderNames.files.find((f) => f.fileName === destFileName); - if (conflictingFileName && conflictResolution === 'skip') { - return emptyArFSResult; + if (conflictingFileName) { + if (conflictResolution === 'skip') { + // File has the same name, skip the upload + return emptyArFSResult; + } + + if ( + conflictResolution === 'upsert' && + conflictingFileName.lastModifiedDate === wrappedFile.lastModifiedDate + ) { + // These files have the same name and last modified date, skip the upload + return emptyArFSResult; + } + + // TODO: Handle this.conflictResolution === 'ask' PE-639 } - // TODO: Handle this.conflictResolution === 'upsert' and 'ask' PE-638 // File is a new revision if destination name conflicts // with an existing file in the destination folder @@ -581,7 +593,7 @@ export class ArDrive extends ArDriveAnonymous { parentFolderId, wrappedFolder, destParentFolderName, - conflictResolution = 'skip' + conflictResolution = 'upsert' }: BulkPublicUploadParams): Promise { const driveId = await this.arFsDao.getDriveIdForFolderId(parentFolderId); @@ -694,43 +706,49 @@ export class ArDrive extends ArDriveAnonymous { // Upload all files in the folder for await (const wrappedFile of wrappedFolder.files) { - // Don't upload this file if there is an existing file and conflict resolution is skip - if (!(conflictResolution === 'skip' && wrappedFile.existingId)) { - const fileDataRewardSettings = { - reward: wrappedFile.getBaseCosts().fileDataBaseReward, - feeMultiple: this.feeMultiple - }; - - const metadataRewardSettings = { - reward: wrappedFile.getBaseCosts().metaDataBaseReward, - feeMultiple: this.feeMultiple - }; - - const uploadFileResult = await this.arFsDao.uploadPublicFile({ - parentFolderId: folderId, - wrappedFile, - driveId, - fileDataRewardSettings, - metadataRewardSettings, - existingFileId: wrappedFile.existingId - }); - - // Capture all file results - uploadEntityFees = { - ...uploadEntityFees, - [uploadFileResult.dataTrxId]: +uploadFileResult.dataTrxReward, - [uploadFileResult.metaDataTrxId]: +uploadFileResult.metaDataTrxReward - }; - uploadEntityResults = [ - ...uploadEntityResults, - { - type: 'file', - metadataTxId: uploadFileResult.metaDataTrxId, - dataTxId: uploadFileResult.dataTrxId, - entityId: uploadFileResult.fileId - } - ]; + if ( + // Conflict resolution is set to skip and there is an existing file + (conflictResolution === 'skip' && wrappedFile.existingId) || + // Conflict resolution is set to upsert and an existing file has the same last modified date + (conflictResolution === 'upsert' && wrappedFile.hasSameLastModifiedDate) + ) { + // Continue loop, don't upload this file + continue; } + const fileDataRewardSettings = { + reward: wrappedFile.getBaseCosts().fileDataBaseReward, + feeMultiple: this.feeMultiple + }; + + const metadataRewardSettings = { + reward: wrappedFile.getBaseCosts().metaDataBaseReward, + feeMultiple: this.feeMultiple + }; + + const uploadFileResult = await this.arFsDao.uploadPublicFile({ + parentFolderId: folderId, + wrappedFile, + driveId, + fileDataRewardSettings, + metadataRewardSettings, + existingFileId: wrappedFile.existingId + }); + + // Capture all file results + uploadEntityFees = { + ...uploadEntityFees, + [uploadFileResult.dataTrxId]: +uploadFileResult.dataTrxReward, + [uploadFileResult.metaDataTrxId]: +uploadFileResult.metaDataTrxReward + }; + uploadEntityResults = [ + ...uploadEntityResults, + { + type: 'file', + metadataTxId: uploadFileResult.metaDataTrxId, + dataTxId: uploadFileResult.dataTrxId, + entityId: uploadFileResult.fileId + } + ]; } // Upload folders, and children of those folders @@ -775,7 +793,7 @@ export class ArDrive extends ArDriveAnonymous { wrappedFile, driveKey, destinationFileName, - conflictResolution + conflictResolution = 'upsert' }: UploadPrivateFileParams): Promise { const driveId = await this.arFsDao.getDriveIdForFolderId(parentFolderId); @@ -794,10 +812,22 @@ export class ArDrive extends ArDriveAnonymous { const conflictingFileName = filesAndFolderNames.files.find((f) => f.fileName === destFileName); - if (conflictingFileName && conflictResolution === 'skip') { - return emptyArFSResult; + if (conflictingFileName) { + if (conflictResolution === 'skip') { + // File has the same name, skip the upload + return emptyArFSResult; + } + + if ( + conflictResolution === 'upsert' && + conflictingFileName.lastModifiedDate === wrappedFile.lastModifiedDate + ) { + // These files have the same name and last modified date, skip the upload + return emptyArFSResult; + } + + // TODO: Handle this.conflictResolution === 'ask' PE-639 } - // TODO: Handle this.conflictResolution === 'upsert' and 'ask' PE-638 // File is a new revision if destination name conflicts // with an existing file in the destination folder @@ -859,7 +889,7 @@ export class ArDrive extends ArDriveAnonymous { wrappedFolder, driveKey, destParentFolderName, - conflictResolution = 'skip' + conflictResolution = 'upsert' }: BulkPrivateUploadParams): Promise { // Retrieve drive ID from folder ID const driveId = await this.arFsDao.getDriveIdForFolderId(parentFolderId); @@ -954,6 +984,11 @@ export class ArDrive extends ArDriveAnonymous { if (fileNameConflict) { // Assigns existing id for later use file.existingId = fileNameConflict.fileId; + + if (fileNameConflict.lastModifiedDate === file.lastModifiedDate) { + // Check last modified date and set to true to resolve upsert conditional + file.hasSameLastModifiedDate = true; + } } } @@ -1055,44 +1090,51 @@ export class ArDrive extends ArDriveAnonymous { // Upload all files in the folder for await (const wrappedFile of wrappedFolder.files) { - // Don't upload this file if there is an existing file and conflict resolution is skip - if (!(conflictResolution === 'skip' && wrappedFile.existingId)) { - const fileDataRewardSettings = { - reward: wrappedFile.getBaseCosts().fileDataBaseReward, - feeMultiple: this.feeMultiple - }; - const metadataRewardSettings = { - reward: wrappedFile.getBaseCosts().metaDataBaseReward, - feeMultiple: this.feeMultiple - }; - - const uploadFileResult = await this.arFsDao.uploadPrivateFile({ - parentFolderId: folderId, - wrappedFile, - driveId, - driveKey, - fileDataRewardSettings, - metadataRewardSettings, - existingFileId: wrappedFile.existingId - }); - - // Capture all file results - uploadEntityFees = { - ...uploadEntityFees, - [uploadFileResult.dataTrxId]: +uploadFileResult.dataTrxReward, - [uploadFileResult.metaDataTrxId]: +uploadFileResult.metaDataTrxReward - }; - uploadEntityResults = [ - ...uploadEntityResults, - { - type: 'file', - metadataTxId: uploadFileResult.metaDataTrxId, - dataTxId: uploadFileResult.dataTrxId, - entityId: uploadFileResult.fileId, - key: urlEncodeHashKey(uploadFileResult.fileKey) - } - ]; + if ( + // Conflict resolution is set to skip and there is an existing file + (conflictResolution === 'skip' && wrappedFile.existingId) || + // Conflict resolution is set to upsert and an existing file has the same last modified date + (conflictResolution === 'upsert' && wrappedFile.hasSameLastModifiedDate) + ) { + // Continue loop, don't upload this file + continue; } + + const fileDataRewardSettings = { + reward: wrappedFile.getBaseCosts().fileDataBaseReward, + feeMultiple: this.feeMultiple + }; + const metadataRewardSettings = { + reward: wrappedFile.getBaseCosts().metaDataBaseReward, + feeMultiple: this.feeMultiple + }; + + const uploadFileResult = await this.arFsDao.uploadPrivateFile({ + parentFolderId: folderId, + wrappedFile, + driveId, + driveKey, + fileDataRewardSettings, + metadataRewardSettings, + existingFileId: wrappedFile.existingId + }); + + // Capture all file results + uploadEntityFees = { + ...uploadEntityFees, + [uploadFileResult.dataTrxId]: +uploadFileResult.dataTrxReward, + [uploadFileResult.metaDataTrxId]: +uploadFileResult.metaDataTrxReward + }; + uploadEntityResults = [ + ...uploadEntityResults, + { + type: 'file', + metadataTxId: uploadFileResult.metaDataTrxId, + dataTxId: uploadFileResult.dataTrxId, + entityId: uploadFileResult.fileId, + key: urlEncodeHashKey(uploadFileResult.fileKey) + } + ]; } // Upload folders, and children of those folders diff --git a/src/arfs_file_wrapper.ts b/src/arfs_file_wrapper.ts index 893a406d..a8e76128 100644 --- a/src/arfs_file_wrapper.ts +++ b/src/arfs_file_wrapper.ts @@ -61,15 +61,20 @@ export class ArFSFileToUpload { baseCosts?: BulkFileBaseCosts; existingId?: FileID; + hasSameLastModifiedDate = false; public gatherFileInfo(): FileInfo { const dataContentType = this.getContentType(); - const lastModifiedDateMS = Math.floor(this.fileStats.mtimeMs); + const lastModifiedDateMS = this.lastModifiedDate; const fileSize = this.fileStats.size; return { dataContentType, lastModifiedDateMS, fileSize }; } + public get lastModifiedDate(): UnixTime { + return Math.floor(this.fileStats.mtimeMs); + } + public getBaseCosts(): BulkFileBaseCosts { if (!this.baseCosts) { throw new Error('Base costs on file were never set!'); diff --git a/src/arfsdao.ts b/src/arfsdao.ts index ba5eaad4..b71adf84 100644 --- a/src/arfsdao.ts +++ b/src/arfsdao.ts @@ -86,7 +86,12 @@ import { ArFSDAOAnonymous } from './arfsdao_anonymous'; import { ArFSFileOrFolderBuilder } from './utils/arfs_builders/arfs_builders'; import { PrivateKeyData } from './private_key_data'; import { ArweaveAddress } from './arweave_address'; -import { EntityNamesAndIds, entityToNameMap, fileToNameAndIdMap, folderToNameAndIdMap } from './utils/mapper_functions'; +import { + EntityNamesAndIds, + entityToNameMap, + fileConflictInfoMap, + folderToNameAndIdMap +} from './utils/mapper_functions'; import { ListPrivateFolderParams } from './ardrive'; export const graphQLURL = 'https://arweave.net/graphql'; @@ -858,7 +863,7 @@ export class ArFSDAO extends ArFSDAOAnonymous { async getPublicEntityNamesAndIdsInFolder(folderId: FolderID): Promise { const childrenOfFolder = await this.getPublicEntitiesInFolder(folderId, true); return { - files: childrenOfFolder.filter(fileFilter).map(fileToNameAndIdMap), + files: childrenOfFolder.filter(fileFilter).map(fileConflictInfoMap), folders: childrenOfFolder.filter(folderFilter).map(folderToNameAndIdMap) }; } @@ -866,7 +871,7 @@ export class ArFSDAO extends ArFSDAOAnonymous { async getPrivateEntityNamesAndIdsInFolder(folderId: FolderID, driveKey: DriveKey): Promise { const childrenOfFolder = await this.getPrivateEntitiesInFolder(folderId, driveKey, true); return { - files: childrenOfFolder.filter(fileFilter).map(fileToNameAndIdMap), + files: childrenOfFolder.filter(fileFilter).map(fileConflictInfoMap), folders: childrenOfFolder.filter(folderFilter).map(folderToNameAndIdMap) }; } diff --git a/src/parameter_declarations.ts b/src/parameter_declarations.ts index 0e9c2d17..abcdddf6 100644 --- a/src/parameter_declarations.ts +++ b/src/parameter_declarations.ts @@ -26,7 +26,7 @@ export const BoostParameter = 'boost'; export const DryRunParameter = 'dryRun'; export const SkipParameter = 'skip'; export const ReplaceParameter = 'replace'; -// export const UpsertParameter = 'upsert'; +export const UpsertParameter = 'upsert'; // export const AskParameter = 'ask'; export const NoVerifyParameter = 'verify'; // commander maps --no-x style params to options.x and always includes in options @@ -226,7 +226,7 @@ Parameter.declare({ aliases: ['--skip'], description: '(OPTIONAL) Skip upload if there is a name conflict within destination folder', type: 'boolean', - forbiddenConjunctionParameters: [ReplaceParameter] + forbiddenConjunctionParameters: [ReplaceParameter, UpsertParameter] }); Parameter.declare({ @@ -234,7 +234,16 @@ Parameter.declare({ aliases: ['--replace'], description: '(OPTIONAL) Create new file revisions if there is a name conflict within destination folder', type: 'boolean', - forbiddenConjunctionParameters: [SkipParameter] + forbiddenConjunctionParameters: [SkipParameter, UpsertParameter] +}); + +Parameter.declare({ + name: UpsertParameter, + aliases: ['--upsert'], + description: + '(OPTIONAL) If there is a name conflict within the destination folder and if that file was last modified at the same time as the file to upload, skip the upload', + type: 'boolean', + forbiddenConjunctionParameters: [SkipParameter, ReplaceParameter] }); Parameter.declare({ diff --git a/src/utils/mapper_functions.ts b/src/utils/mapper_functions.ts index d77cbb91..1cc03f46 100644 --- a/src/utils/mapper_functions.ts +++ b/src/utils/mapper_functions.ts @@ -1,8 +1,8 @@ import { ArFSFileOrFolderEntity } from '../arfs_entities'; -import { FileID, FolderID } from '../types'; +import { FileID, FolderID, UnixTime } from '../types'; export interface EntityNamesAndIds { - files: FileNameAndId[]; + files: FileConflictInfo[]; folders: FolderNameAndId[]; } @@ -11,9 +11,10 @@ interface FolderNameAndId { folderId: FolderID; } -interface FileNameAndId { +interface FileConflictInfo { fileName: string; fileId: FileID; + lastModifiedDate: UnixTime; } export function entityToNameMap(entity: ArFSFileOrFolderEntity): string { @@ -24,6 +25,6 @@ export function folderToNameAndIdMap(entity: ArFSFileOrFolderEntity): FolderName return { folderId: entity.entityId, folderName: entity.name }; } -export function fileToNameAndIdMap(entity: ArFSFileOrFolderEntity): FileNameAndId { - return { fileId: entity.entityId, fileName: entity.name }; +export function fileConflictInfoMap(entity: ArFSFileOrFolderEntity): FileConflictInfo { + return { fileId: entity.entityId, fileName: entity.name, lastModifiedDate: entity.lastModifiedDate }; } From 7613f0dee2b6d833c361f8f91923aebb3b24eb26 Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Fri, 29 Oct 2021 20:30:12 -0500 Subject: [PATCH 10/31] test(upsert): Add cases for upsert PE-633 --- tests/integration/ardrive.int.test.ts | 87 ++++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 2 deletions(-) diff --git a/tests/integration/ardrive.int.test.ts b/tests/integration/ardrive.int.test.ts index fa974975..8702e660 100644 --- a/tests/integration/ardrive.int.test.ts +++ b/tests/integration/ardrive.int.test.ts @@ -451,6 +451,9 @@ describe('ArDrive class - integrated', () => { }); describe('file function', () => { + const matchingLastModifiedDate = 420; + const differentLastModifiedDate = 1337; + describe('uploadPublicFile', () => { const wrappedFile = wrapFileOrFolder('test_wallet.json') as ArFSFileToUpload; @@ -459,7 +462,13 @@ describe('ArDrive class - integrated', () => { stub(communityOracle, 'selectTokenHolder').resolves(stubArweaveAddress()); stub(arfsDao, 'getPublicEntityNamesAndIdsInFolder').resolves({ - files: [{ fileName: 'CONFLICTING_FILE_NAME', fileId: existingFileId }], + files: [ + { + fileName: 'CONFLICTING_FILE_NAME', + fileId: existingFileId, + lastModifiedDate: matchingLastModifiedDate + } + ], folders: [{ folderName: 'CONFLICTING_FOLDER_NAME', folderId: stubEntityID }] }); }); @@ -517,6 +526,39 @@ describe('ArDrive class - integrated', () => { assertUploadFileExpectations(result, 3204, 171, 0, '1', 'public', existingFileId); }); + it('returns an empty ArFSResult if destination folder has a conflicting FILE name and a matching last modified date and the conflict resolution is set to upsert', async () => { + stub(arfsDao, 'getOwnerForDriveId').resolves(walletOwner); + stub(wrappedFile, 'lastModifiedDate').get(() => matchingLastModifiedDate); + + const result = await arDrive.uploadPublicFile({ + parentFolderId: stubEntityID, + wrappedFile, + destinationFileName: 'CONFLICTING_FILE_NAME', + conflictResolution: 'upsert' + }); + + expect(result).to.deep.equal({ + created: [], + tips: [], + fees: {} + }); + }); + + it('returns the correct ArFSResult revision if destination folder has a conflicting FILE name and a different last modified date and the conflict resolution is set to upsert', async () => { + stub(arfsDao, 'getOwnerForDriveId').resolves(walletOwner); + stub(wrappedFile, 'lastModifiedDate').get(() => differentLastModifiedDate); + + const result = await arDrive.uploadPublicFile({ + parentFolderId: stubEntityID, + wrappedFile, + destinationFileName: 'CONFLICTING_FILE_NAME', + conflictResolution: 'upsert' + }); + + // Pass expected existing file id, so that the file would be considered a revision + assertUploadFileExpectations(result, 3204, 162, 0, '1', 'public', existingFileId); + }); + it('returns the correct ArFSResult', async () => { stub(arfsDao, 'getOwnerForDriveId').resolves(walletOwner); @@ -533,7 +575,13 @@ describe('ArDrive class - integrated', () => { stub(communityOracle, 'selectTokenHolder').resolves(stubArweaveAddress()); stub(arfsDao, 'getPrivateEntityNamesAndIdsInFolder').resolves({ - files: [{ fileName: 'CONFLICTING_FILE_NAME', fileId: existingFileId }], + files: [ + { + fileName: 'CONFLICTING_FILE_NAME', + fileId: existingFileId, + lastModifiedDate: matchingLastModifiedDate + } + ], folders: [{ folderName: 'CONFLICTING_FOLDER_NAME', folderId: stubEntityID }] }); }); @@ -598,6 +646,41 @@ describe('ArDrive class - integrated', () => { assertUploadFileExpectations(result, 3216, 187, 0, '1', 'private', existingFileId); }); + it('returns an empty ArFSResult if destination folder has a conflicting FILE name and a matching last modified date and the conflict resolution is set to upsert', async () => { + stub(arfsDao, 'getOwnerForDriveId').resolves(walletOwner); + stub(wrappedFile, 'lastModifiedDate').get(() => matchingLastModifiedDate); + + const result = await arDrive.uploadPrivateFile({ + parentFolderId: stubEntityID, + wrappedFile, + destinationFileName: 'CONFLICTING_FILE_NAME', + conflictResolution: 'upsert', + driveKey: await getStubDriveKey() + }); + + expect(result).to.deep.equal({ + created: [], + tips: [], + fees: {} + }); + }); + + it('returns the correct ArFSResult revision if destination folder has a conflicting FILE name and a different last modified date and the conflict resolution is set to upsert', async () => { + stub(arfsDao, 'getOwnerForDriveId').resolves(walletOwner); + stub(wrappedFile, 'lastModifiedDate').get(() => differentLastModifiedDate); + + const result = await arDrive.uploadPrivateFile({ + parentFolderId: stubEntityID, + wrappedFile, + destinationFileName: 'CONFLICTING_FILE_NAME', + conflictResolution: 'upsert', + driveKey: await getStubDriveKey() + }); + + // Pass expected existing file id, so that the file would be considered a revision + assertUploadFileExpectations(result, 3216, 178, 0, '1', 'private', existingFileId); + }); + it('returns the correct ArFSResult', async () => { stub(arfsDao, 'getOwnerForDriveId').resolves(walletOwner); const stubDriveKey = await getStubDriveKey(); From 33d161c8a22d202d78195767435100f97dae0150 Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Mon, 1 Nov 2021 11:04:41 -0500 Subject: [PATCH 11/31] docs(upsert): Update README with upsert PE-633 --- README.md | 8 +++++--- src/parameter_declarations.ts | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e6e60df5..bcac55be 100644 --- a/README.md +++ b/README.md @@ -690,7 +690,9 @@ NOTE: To upload to the root of a drive, specify its root folder ID as the parent ardrive drive-info -d "c7f87712-b54e-4491-bc96-1c5fa7b1da50" | jq -r '.rootFolderId' ``` -By default, the single `upload-file` command will skip name conflicts found. To override this behavior and make a new revision of a file, use the `--replace` option: +By default, the single `upload-file` command will `--upsert` on name conflicts found. This means that when it finds a file in the destination folder with the same name, it will compare the last modified dates of that file and the file to upload. If they are matching, the upload will be skipped, otherwise the upload will be added as a new revision. + +To override this behavior, use the `--replace` option to always make new revisions of a file or the `--skip` option to always skip the upload on name conflicts: ```shell ardrive upload-file --replace --local-file-path /path/to/file.txt --parent-folder-id "9af694f6-4cfc-4eee-88a8-1b02704760c0" -w /path/to/wallet.json @@ -709,9 +711,9 @@ This method of upload can be used to upload a large number of files and folders - Folder names that conflict with a FILE name at the destination will cause an error to be thrown - Folder names that conflict with a FOLDER name at the destination will use the existing folder ID (i.e. skip) rather than creating a new folder - File names that conflict with a FOLDER name at the destination will cause an error to be thrown -- File names that conflict with a FILE name at the destination will be SKIPPED +- When the intended file name conflicts with a FILE name at the destination the file will be SKIPPED if the last modified date matches the file to upload. If they have different last modified dates, it will be uploaded as a new REVISION -Similar to the single file upload, the above FILE to FILE name conflict resolution behavior can be modified by the `--replace` command. This will force new revisions on all conflicts within the bulk upload. +Similar to the single file upload, the above FILE to FILE name conflict resolution behavior can be modified. Use the `--replace` option to will force new revisions on all conflicts within the bulk upload regardless of last modified date. Or use the `--skip` option to simply skip all FILE to FILE name conflicts. ### Fetching the Metadata of a File Entity diff --git a/src/parameter_declarations.ts b/src/parameter_declarations.ts index abcdddf6..9fd823d5 100644 --- a/src/parameter_declarations.ts +++ b/src/parameter_declarations.ts @@ -241,7 +241,7 @@ Parameter.declare({ name: UpsertParameter, aliases: ['--upsert'], description: - '(OPTIONAL) If there is a name conflict within the destination folder and if that file was last modified at the same time as the file to upload, skip the upload', + '(OPTIONAL) When there is a name conflict within the destination folder, if that file was last modified at the same time as the file to upload, skip the upload, otherwise upload that file as a new revision', type: 'boolean', forbiddenConjunctionParameters: [SkipParameter, ReplaceParameter] }); From 305b3b1d3712e96f948462b299b3e36268c1a09a Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Mon, 1 Nov 2021 11:28:25 -0500 Subject: [PATCH 12/31] refactor(upsert): Rename method for accuracy PE-633 --- src/ardrive.ts | 12 ++++++------ src/arfsdao.ts | 4 ++-- tests/integration/ardrive.int.test.ts | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/ardrive.ts b/src/ardrive.ts index 7de28902..d0455171 100644 --- a/src/ardrive.ts +++ b/src/ardrive.ts @@ -518,7 +518,7 @@ export class ArDrive extends ArDriveAnonymous { // Derive destination name and names already within provided destination folder const destFileName = destinationFileName ?? wrappedFile.getBaseFileName(); - const filesAndFolderNames = await this.arFsDao.getPublicEntityNamesAndIdsInFolder(parentFolderId); + const filesAndFolderNames = await this.arFsDao.getPublicNameConflictInfoInFolder(parentFolderId); // Files cannot overwrite folder names if (filesAndFolderNames.folders.find((f) => f.folderName === destFileName)) { @@ -602,7 +602,7 @@ export class ArDrive extends ArDriveAnonymous { // Derive destination name and names already within provided destination folder const destFolderName = destParentFolderName ?? wrappedFolder.getBaseFileName(); - const filesAndFolderNames = await this.arFsDao.getPublicEntityNamesAndIdsInFolder(parentFolderId); + const filesAndFolderNames = await this.arFsDao.getPublicNameConflictInfoInFolder(parentFolderId); // Folders cannot overwrite file names if (filesAndFolderNames.files.find((f) => f.fileName === destFolderName)) { @@ -802,7 +802,7 @@ export class ArDrive extends ArDriveAnonymous { // Derive destination name and names already within provided destination folder const destFileName = destinationFileName ?? wrappedFile.getBaseFileName(); - const filesAndFolderNames = await this.arFsDao.getPrivateEntityNamesAndIdsInFolder(parentFolderId, driveKey); + const filesAndFolderNames = await this.arFsDao.getPrivateNameConflictInfoInFolder(parentFolderId, driveKey); // Files cannot overwrite folder names if (filesAndFolderNames.folders.find((f) => f.folderName === destFileName)) { @@ -902,7 +902,7 @@ export class ArDrive extends ArDriveAnonymous { // Derive destination name and names already within provided destination folder const destFolderName = destParentFolderName ?? wrappedFolder.getBaseFileName(); - const filesAndFolderNames = await this.arFsDao.getPrivateEntityNamesAndIdsInFolder(parentFolderId, driveKey); + const filesAndFolderNames = await this.arFsDao.getPrivateNameConflictInfoInFolder(parentFolderId, driveKey); // Folders cannot overwrite file names if (filesAndFolderNames.files.find((f) => f.fileName === destFolderName)) { @@ -1021,7 +1021,7 @@ export class ArDrive extends ArDriveAnonymous { protected async checkAndAssignExistingPublicNames(wrappedFolder: ArFSFolderToUpload): Promise { await this.checkAndAssignExistingNames(wrappedFolder, (parentFolderId) => - this.arFsDao.getPublicEntityNamesAndIdsInFolder(parentFolderId) + this.arFsDao.getPublicNameConflictInfoInFolder(parentFolderId) ); } @@ -1030,7 +1030,7 @@ export class ArDrive extends ArDriveAnonymous { driveKey: DriveKey ): Promise { await this.checkAndAssignExistingNames(wrappedFolder, (parentFolderId) => - this.arFsDao.getPrivateEntityNamesAndIdsInFolder(parentFolderId, driveKey) + this.arFsDao.getPrivateNameConflictInfoInFolder(parentFolderId, driveKey) ); } diff --git a/src/arfsdao.ts b/src/arfsdao.ts index b71adf84..89636c14 100644 --- a/src/arfsdao.ts +++ b/src/arfsdao.ts @@ -860,7 +860,7 @@ export class ArFSDAO extends ArFSDAOAnonymous { return childrenOfFolder.map(entityToNameMap); } - async getPublicEntityNamesAndIdsInFolder(folderId: FolderID): Promise { + async getPublicNameConflictInfoInFolder(folderId: FolderID): Promise { const childrenOfFolder = await this.getPublicEntitiesInFolder(folderId, true); return { files: childrenOfFolder.filter(fileFilter).map(fileConflictInfoMap), @@ -868,7 +868,7 @@ export class ArFSDAO extends ArFSDAOAnonymous { }; } - async getPrivateEntityNamesAndIdsInFolder(folderId: FolderID, driveKey: DriveKey): Promise { + async getPrivateNameConflictInfoInFolder(folderId: FolderID, driveKey: DriveKey): Promise { const childrenOfFolder = await this.getPrivateEntitiesInFolder(folderId, driveKey, true); return { files: childrenOfFolder.filter(fileFilter).map(fileConflictInfoMap), diff --git a/tests/integration/ardrive.int.test.ts b/tests/integration/ardrive.int.test.ts index 8702e660..a94e645a 100644 --- a/tests/integration/ardrive.int.test.ts +++ b/tests/integration/ardrive.int.test.ts @@ -461,7 +461,7 @@ describe('ArDrive class - integrated', () => { stub(communityOracle, 'getCommunityWinstonTip').resolves('1'); stub(communityOracle, 'selectTokenHolder').resolves(stubArweaveAddress()); - stub(arfsDao, 'getPublicEntityNamesAndIdsInFolder').resolves({ + stub(arfsDao, 'getPublicNameConflictInfoInFolder').resolves({ files: [ { fileName: 'CONFLICTING_FILE_NAME', @@ -574,7 +574,7 @@ describe('ArDrive class - integrated', () => { stub(communityOracle, 'getCommunityWinstonTip').resolves('1'); stub(communityOracle, 'selectTokenHolder').resolves(stubArweaveAddress()); - stub(arfsDao, 'getPrivateEntityNamesAndIdsInFolder').resolves({ + stub(arfsDao, 'getPrivateNameConflictInfoInFolder').resolves({ files: [ { fileName: 'CONFLICTING_FILE_NAME', From 48bb794ff41542a8572b87b3e29dad6cc43dd2cf Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Mon, 1 Nov 2021 20:01:04 -0500 Subject: [PATCH 13/31] docs(upsert): Rephrase upsert description PE-649 --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index bcac55be..dd7b15db 100644 --- a/README.md +++ b/README.md @@ -690,9 +690,11 @@ NOTE: To upload to the root of a drive, specify its root folder ID as the parent ardrive drive-info -d "c7f87712-b54e-4491-bc96-1c5fa7b1da50" | jq -r '.rootFolderId' ``` -By default, the single `upload-file` command will `--upsert` on name conflicts found. This means that when it finds a file in the destination folder with the same name, it will compare the last modified dates of that file and the file to upload. If they are matching, the upload will be skipped, otherwise the upload will be added as a new revision. +By default, the single `upload-file` command will use the upsert behavior. It will check the destination folder for a file with a conflicting name. If no conflicts are found, it will insert (upload) the file. -To override this behavior, use the `--replace` option to always make new revisions of a file or the `--skip` option to always skip the upload on name conflicts: +In the case that there is a FILE to FILE name conflict found, it will only update it if necessary. To determine if an update is necessary, upsert will compare the last modified dates of conflicting file and the file being uploaded. When they are matching, the upload will be skipped. Otherwise the file will be updated as a new revision. + +To override the upsert behavior, use the `--replace` option to always make new revisions of a file or the `--skip` option to always skip the upload on name conflicts: ```shell ardrive upload-file --replace --local-file-path /path/to/file.txt --parent-folder-id "9af694f6-4cfc-4eee-88a8-1b02704760c0" -w /path/to/wallet.json @@ -713,7 +715,7 @@ This method of upload can be used to upload a large number of files and folders - File names that conflict with a FOLDER name at the destination will cause an error to be thrown - When the intended file name conflicts with a FILE name at the destination the file will be SKIPPED if the last modified date matches the file to upload. If they have different last modified dates, it will be uploaded as a new REVISION -Similar to the single file upload, the above FILE to FILE name conflict resolution behavior can be modified. Use the `--replace` option to will force new revisions on all conflicts within the bulk upload regardless of last modified date. Or use the `--skip` option to simply skip all FILE to FILE name conflicts. +Similar to the single file upload, the above FILE to FILE name conflict resolution behavior can be modified. Use the `--replace` option to force new revisions on all conflicts within the bulk upload regardless of last modified date. Or use the `--skip` option to simply skip all FILE to FILE and/or FILE to FOLDER name conflicts. ### Fetching the Metadata of a File Entity From d010dbcd1070f474413dfc82b8792e7bb6d3e582 Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Mon, 1 Nov 2021 20:20:45 -0500 Subject: [PATCH 14/31] refactor(skip): Skip with empty result on single upload file to folder conflict PE-633 --- src/ardrive.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/ardrive.ts b/src/ardrive.ts index d0455171..f5f27e13 100644 --- a/src/ardrive.ts +++ b/src/ardrive.ts @@ -522,6 +522,11 @@ export class ArDrive extends ArDriveAnonymous { // Files cannot overwrite folder names if (filesAndFolderNames.folders.find((f) => f.folderName === destFileName)) { + if (conflictResolution === 'skip') { + // Return empty result if resolution set to skip on FILE to FOLDER name conflicts + return emptyArFSResult; + } + // TODO: Add optional interactive prompt to resolve name conflicts in ticket PE-599 throw new Error(errorMessage.entityNameExists); } @@ -806,6 +811,11 @@ export class ArDrive extends ArDriveAnonymous { // Files cannot overwrite folder names if (filesAndFolderNames.folders.find((f) => f.folderName === destFileName)) { + if (conflictResolution === 'skip') { + // Return empty result if resolution set to skip on FILE to FOLDER name conflicts + return emptyArFSResult; + } + // TODO: Add optional interactive prompt to resolve name conflicts in ticket PE-599 throw new Error(errorMessage.entityNameExists); } From fb3e029b863fab05be58cdf0fef0d238600cfcd2 Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Mon, 1 Nov 2021 20:36:30 -0500 Subject: [PATCH 15/31] refactor(skip): Skip conflicts on bulk upload PE-633 --- src/ardrive.ts | 88 +++++++++------------------------------- src/arfs_file_wrapper.ts | 67 ++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 68 deletions(-) diff --git a/src/ardrive.ts b/src/ardrive.ts index f5f27e13..48b38a42 100644 --- a/src/ardrive.ts +++ b/src/ardrive.ts @@ -45,7 +45,6 @@ import { import { stubEntityID, stubTransactionID } from './utils/stubs'; import { errorMessage } from './error_message'; import { PrivateKeyData } from './private_key_data'; -import { EntityNamesAndIds } from './utils/mapper_functions'; import { ArweaveAddress } from './arweave_address'; import { WithDriveKey } from './arfs_entity_result_factory'; @@ -965,72 +964,8 @@ export class ArDrive extends ArDriveAnonymous { }); } - protected async checkAndAssignExistingNames( - wrappedFolder: ArFSFolderToUpload, - getExistingNamesFn: (parentFolderId: FolderID) => Promise - ): Promise { - if (!wrappedFolder.existingId) { - // Folder has no existing ID to check - return; - } - - const existingEntityNamesAndIds = await getExistingNamesFn(wrappedFolder.existingId); - - for await (const file of wrappedFolder.files) { - const baseFileName = file.getBaseFileName(); - - const folderNameConflict = existingEntityNamesAndIds.folders.find( - ({ folderName }) => folderName === baseFileName - ); - - // File name cannot conflict with a folder name - if (folderNameConflict) { - throw new Error(errorMessage.entityNameExists); - } - - const fileNameConflict = existingEntityNamesAndIds.files.find(({ fileName }) => fileName === baseFileName); - - // Conflicting file name creates a REVISION by default - if (fileNameConflict) { - // Assigns existing id for later use - file.existingId = fileNameConflict.fileId; - - if (fileNameConflict.lastModifiedDate === file.lastModifiedDate) { - // Check last modified date and set to true to resolve upsert conditional - file.hasSameLastModifiedDate = true; - } - } - } - - for await (const folder of wrappedFolder.folders) { - const baseFolderName = folder.getBaseFileName(); - - const fileNameConflict = existingEntityNamesAndIds.files.find( - ({ fileName }) => fileName === baseFolderName - ); - - // Folder name cannot conflict with a file name - if (fileNameConflict) { - throw new Error(errorMessage.entityNameExists); - } - - const folderNameConflict = existingEntityNamesAndIds.folders.find( - ({ folderName }) => folderName === baseFolderName - ); - - // Conflicting folder name uses EXISTING folder by default - if (folderNameConflict) { - // Assigns existing id for later use - folder.existingId = folderNameConflict.folderId; - - // Recurse into existing folder on folder name conflict - await this.checkAndAssignExistingNames(folder, getExistingNamesFn); - } - } - } - protected async checkAndAssignExistingPublicNames(wrappedFolder: ArFSFolderToUpload): Promise { - await this.checkAndAssignExistingNames(wrappedFolder, (parentFolderId) => + await wrappedFolder.checkAndAssignExistingNames((parentFolderId) => this.arFsDao.getPublicNameConflictInfoInFolder(parentFolderId) ); } @@ -1039,7 +974,7 @@ export class ArDrive extends ArDriveAnonymous { wrappedFolder: ArFSFolderToUpload, driveKey: DriveKey ): Promise { - await this.checkAndAssignExistingNames(wrappedFolder, (parentFolderId) => + await wrappedFolder.checkAndAssignExistingNames((parentFolderId) => this.arFsDao.getPrivateNameConflictInfoInFolder(parentFolderId, driveKey) ); } @@ -1059,7 +994,14 @@ export class ArDrive extends ArDriveAnonymous { let uploadEntityResults: ArFSEntityData[] = []; let folderId: FolderID; - if (wrappedFolder.existingId) { + if (wrappedFolder.fileNameConflict) { + if (conflictResolution === 'skip') { + // Return empty result on skip + return { entityResults: uploadEntityResults, feeResults: uploadEntityFees }; + } + // Otherwise throw an error, folder names cannot conflict with file names + throw new Error(errorMessage.entityNameExists); + } else if (wrappedFolder.existingId) { // Use existing parent folder ID for bulk upload. // This happens when the parent folder's name conflicts folderId = wrappedFolder.existingId; @@ -1110,6 +1052,16 @@ export class ArDrive extends ArDriveAnonymous { continue; } + if (wrappedFile.folderNameConflict) { + if (conflictResolution === 'skip') { + // Continue loop, skip uploading this file + continue; + } + + // Otherwise throw an error, file names cannot conflict with folder names + throw new Error(errorMessage.entityNameExists); + } + const fileDataRewardSettings = { reward: wrappedFile.getBaseCosts().fileDataBaseReward, feeMultiple: this.feeMultiple diff --git a/src/arfs_file_wrapper.ts b/src/arfs_file_wrapper.ts index a8e76128..392f2ac8 100644 --- a/src/arfs_file_wrapper.ts +++ b/src/arfs_file_wrapper.ts @@ -3,6 +3,7 @@ import { extToMime } from 'ardrive-core-js'; import { basename, join } from 'path'; import { ByteCount, DataContentType, FileID, FolderID, UnixTime } from './types'; import { BulkFileBaseCosts, MetaDataBaseCosts } from './ardrive'; +import { EntityNamesAndIds } from './utils/mapper_functions'; type BaseFileName = string; type FilePath = string; @@ -61,6 +62,7 @@ export class ArFSFileToUpload { baseCosts?: BulkFileBaseCosts; existingId?: FileID; + folderNameConflict = false; hasSameLastModifiedDate = false; public gatherFileInfo(): FileInfo { @@ -107,6 +109,7 @@ export class ArFSFolderToUpload { baseCosts?: MetaDataBaseCosts; existingId?: FolderID; destinationName?: string; + fileNameConflict = false; constructor(public readonly filePath: FilePath, public readonly fileStats: fs.Stats) { const entitiesInFolder = fs.readdirSync(this.filePath); @@ -127,6 +130,70 @@ export class ArFSFolderToUpload { } } + public async checkAndAssignExistingNames( + getExistingNamesFn: (parentFolderId: FolderID) => Promise + ): Promise { + if (!this.existingId) { + // Folder has no existing ID to check + return; + } + + const existingEntityNamesAndIds = await getExistingNamesFn(this.existingId); + + for await (const file of this.files) { + const baseFileName = file.getBaseFileName(); + + const folderNameConflict = existingEntityNamesAndIds.folders.find( + ({ folderName }) => folderName === baseFileName + ); + + if (folderNameConflict) { + // Folder name cannot conflict with a file name + file.folderNameConflict = true; + continue; + } + + const fileNameConflict = existingEntityNamesAndIds.files.find(({ fileName }) => fileName === baseFileName); + + // Conflicting file name creates a REVISION by default + if (fileNameConflict) { + file.existingId = fileNameConflict.fileId; + + if (fileNameConflict.lastModifiedDate === file.lastModifiedDate) { + // Check last modified date and set to true to resolve upsert conditional + file.hasSameLastModifiedDate = true; + } + } + } + + for await (const folder of this.folders) { + const baseFolderName = folder.getBaseFileName(); + + const fileNameConflict = existingEntityNamesAndIds.files.find( + ({ fileName }) => fileName === baseFolderName + ); + + if (fileNameConflict) { + // Folder name cannot conflict with a file name + this.fileNameConflict = true; + continue; + } + + const folderNameConflict = existingEntityNamesAndIds.folders.find( + ({ folderName }) => folderName === baseFolderName + ); + + // Conflicting folder name uses EXISTING folder by default + if (folderNameConflict) { + // Assigns existing id for later use + folder.existingId = folderNameConflict.folderId; + + // Recurse into existing folder on folder name conflict + await folder.checkAndAssignExistingNames(getExistingNamesFn); + } + } + } + public getBaseCosts(): MetaDataBaseCosts { if (!this.baseCosts) { throw new Error('Base costs on folder were never set!'); From c8cb8cd0b9c6ec0d87af8cb5f429be35db6fb115 Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Mon, 1 Nov 2021 20:38:05 -0500 Subject: [PATCH 16/31] docs(skip): Update readme with expected behavior PE-649 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dd7b15db..b1595a38 100644 --- a/README.md +++ b/README.md @@ -715,7 +715,7 @@ This method of upload can be used to upload a large number of files and folders - File names that conflict with a FOLDER name at the destination will cause an error to be thrown - When the intended file name conflicts with a FILE name at the destination the file will be SKIPPED if the last modified date matches the file to upload. If they have different last modified dates, it will be uploaded as a new REVISION -Similar to the single file upload, the above FILE to FILE name conflict resolution behavior can be modified. Use the `--replace` option to force new revisions on all conflicts within the bulk upload regardless of last modified date. Or use the `--skip` option to simply skip all FILE to FILE and/or FILE to FOLDER name conflicts. +Similar to the single file upload, the above FILE to FILE name conflict resolution behavior can be modified. Use the `--replace` option to force new revisions on all conflicts within the bulk upload regardless of last modified date. Or use the `--skip` option to simply skip ALL name conflicts. ### Fetching the Metadata of a File Entity From 7b0d6e093d7efe5f4101b4a53993530a0f994133 Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Mon, 1 Nov 2021 20:41:25 -0500 Subject: [PATCH 17/31] fix(upsert): Uncomment parameter PE-633 --- src/parameter_declarations.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parameter_declarations.ts b/src/parameter_declarations.ts index 9fd823d5..524b0e00 100644 --- a/src/parameter_declarations.ts +++ b/src/parameter_declarations.ts @@ -40,7 +40,7 @@ export const DriveCreationPrivacyParameters = [ export const DrivePrivacyParameters = [DriveKeyParameter, ...DriveCreationPrivacyParameters]; export const TreeDepthParams = [AllParameter, MaxDepthParameter]; -export const ConflictResolutionParams = [SkipParameter, ReplaceParameter /* , UpsertParameter, AskParameter */]; +export const ConflictResolutionParams = [SkipParameter, ReplaceParameter, UpsertParameter /* , AskParameter */]; /** * Note: importing this file will declare all the above parameters From c1f8da05739f08bcf0fd73861c506490c6d8b1a0 Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Mon, 1 Nov 2021 20:50:05 -0500 Subject: [PATCH 18/31] refactor(conflicts): Use const declarations PE-633 --- src/CLICommand/parameters_helper.ts | 10 ++++---- src/ardrive.ts | 40 +++++++++++++++++------------ 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/src/CLICommand/parameters_helper.ts b/src/CLICommand/parameters_helper.ts index c6ac84b9..e530d418 100644 --- a/src/CLICommand/parameters_helper.ts +++ b/src/CLICommand/parameters_helper.ts @@ -19,7 +19,7 @@ import { DriveID, DriveKey } from '../types'; import passwordPrompt from 'prompts'; import { PrivateKeyData } from '../private_key_data'; import { ArweaveAddress } from '../arweave_address'; -import { FileNameConflictResolution } from '../ardrive'; +import { FileNameConflictResolution, replaceOnConflicts, skipOnConflicts, upsertOnConflicts } from '../ardrive'; // eslint-disable-next-line @typescript-eslint/no-explicit-any type ParameterOptions = any; @@ -209,18 +209,18 @@ export class ParametersHelper { public getFileNameConflictResolution(): FileNameConflictResolution { if (this.getParameterValue(ReplaceParameter)) { - return 'replace'; + return replaceOnConflicts; } if (this.getParameterValue(UpsertParameter)) { - return 'skip'; + return skipOnConflicts; } // if (this.getParameterValue(AskParameter)) { - // return 'ask' + // return askOnConflicts; // }; - return 'upsert'; + return upsertOnConflicts; } /** diff --git a/src/ardrive.ts b/src/ardrive.ts index 48b38a42..ec7807e9 100644 --- a/src/ardrive.ts +++ b/src/ardrive.ts @@ -124,7 +124,13 @@ interface MovePublicFolderParams { } type MovePrivateFolderParams = MovePublicFolderParams & WithDriveKey; -export type FileNameConflictResolution = 'skip' | 'replace' | 'upsert'; // | 'ask' +export const skipOnConflicts = 'skip'; +export const replaceOnConflicts = 'replace'; +export const upsertOnConflicts = 'upsert'; +// export const askOnConflicts = 'ask'; + +export type FileNameConflictResolution = typeof skipOnConflicts | typeof replaceOnConflicts | typeof upsertOnConflicts; +// | typeof askOnConflicts; export interface UploadParams { parentFolderId: FolderID; @@ -508,7 +514,7 @@ export class ArDrive extends ArDriveAnonymous { parentFolderId, wrappedFile, destinationFileName, - conflictResolution = 'upsert' + conflictResolution = upsertOnConflicts }: UploadPublicFileParams): Promise { const driveId = await this.arFsDao.getDriveIdForFolderId(parentFolderId); @@ -521,7 +527,7 @@ export class ArDrive extends ArDriveAnonymous { // Files cannot overwrite folder names if (filesAndFolderNames.folders.find((f) => f.folderName === destFileName)) { - if (conflictResolution === 'skip') { + if (conflictResolution === skipOnConflicts) { // Return empty result if resolution set to skip on FILE to FOLDER name conflicts return emptyArFSResult; } @@ -533,13 +539,13 @@ export class ArDrive extends ArDriveAnonymous { const conflictingFileName = filesAndFolderNames.files.find((f) => f.fileName === destFileName); if (conflictingFileName) { - if (conflictResolution === 'skip') { + if (conflictResolution === skipOnConflicts) { // File has the same name, skip the upload return emptyArFSResult; } if ( - conflictResolution === 'upsert' && + conflictResolution === upsertOnConflicts && conflictingFileName.lastModifiedDate === wrappedFile.lastModifiedDate ) { // These files have the same name and last modified date, skip the upload @@ -597,7 +603,7 @@ export class ArDrive extends ArDriveAnonymous { parentFolderId, wrappedFolder, destParentFolderName, - conflictResolution = 'upsert' + conflictResolution = upsertOnConflicts }: BulkPublicUploadParams): Promise { const driveId = await this.arFsDao.getDriveIdForFolderId(parentFolderId); @@ -712,9 +718,9 @@ export class ArDrive extends ArDriveAnonymous { for await (const wrappedFile of wrappedFolder.files) { if ( // Conflict resolution is set to skip and there is an existing file - (conflictResolution === 'skip' && wrappedFile.existingId) || + (conflictResolution === skipOnConflicts && wrappedFile.existingId) || // Conflict resolution is set to upsert and an existing file has the same last modified date - (conflictResolution === 'upsert' && wrappedFile.hasSameLastModifiedDate) + (conflictResolution === upsertOnConflicts && wrappedFile.hasSameLastModifiedDate) ) { // Continue loop, don't upload this file continue; @@ -797,7 +803,7 @@ export class ArDrive extends ArDriveAnonymous { wrappedFile, driveKey, destinationFileName, - conflictResolution = 'upsert' + conflictResolution = upsertOnConflicts }: UploadPrivateFileParams): Promise { const driveId = await this.arFsDao.getDriveIdForFolderId(parentFolderId); @@ -810,7 +816,7 @@ export class ArDrive extends ArDriveAnonymous { // Files cannot overwrite folder names if (filesAndFolderNames.folders.find((f) => f.folderName === destFileName)) { - if (conflictResolution === 'skip') { + if (conflictResolution === skipOnConflicts) { // Return empty result if resolution set to skip on FILE to FOLDER name conflicts return emptyArFSResult; } @@ -822,13 +828,13 @@ export class ArDrive extends ArDriveAnonymous { const conflictingFileName = filesAndFolderNames.files.find((f) => f.fileName === destFileName); if (conflictingFileName) { - if (conflictResolution === 'skip') { + if (conflictResolution === skipOnConflicts) { // File has the same name, skip the upload return emptyArFSResult; } if ( - conflictResolution === 'upsert' && + conflictResolution === upsertOnConflicts && conflictingFileName.lastModifiedDate === wrappedFile.lastModifiedDate ) { // These files have the same name and last modified date, skip the upload @@ -898,7 +904,7 @@ export class ArDrive extends ArDriveAnonymous { wrappedFolder, driveKey, destParentFolderName, - conflictResolution = 'upsert' + conflictResolution = upsertOnConflicts }: BulkPrivateUploadParams): Promise { // Retrieve drive ID from folder ID const driveId = await this.arFsDao.getDriveIdForFolderId(parentFolderId); @@ -995,7 +1001,7 @@ export class ArDrive extends ArDriveAnonymous { let folderId: FolderID; if (wrappedFolder.fileNameConflict) { - if (conflictResolution === 'skip') { + if (conflictResolution === skipOnConflicts) { // Return empty result on skip return { entityResults: uploadEntityResults, feeResults: uploadEntityFees }; } @@ -1044,16 +1050,16 @@ export class ArDrive extends ArDriveAnonymous { for await (const wrappedFile of wrappedFolder.files) { if ( // Conflict resolution is set to skip and there is an existing file - (conflictResolution === 'skip' && wrappedFile.existingId) || + (conflictResolution === skipOnConflicts && wrappedFile.existingId) || // Conflict resolution is set to upsert and an existing file has the same last modified date - (conflictResolution === 'upsert' && wrappedFile.hasSameLastModifiedDate) + (conflictResolution === upsertOnConflicts && wrappedFile.hasSameLastModifiedDate) ) { // Continue loop, don't upload this file continue; } if (wrappedFile.folderNameConflict) { - if (conflictResolution === 'skip') { + if (conflictResolution === skipOnConflicts) { // Continue loop, skip uploading this file continue; } From 02dff25d520af1557ea76635bb9fa8e77119aea3 Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Tue, 2 Nov 2021 10:22:58 -0500 Subject: [PATCH 19/31] refactor(conflicts): Return actual empty result PE-633 --- src/ardrive.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/ardrive.ts b/src/ardrive.ts index ec7807e9..5604935e 100644 --- a/src/ardrive.ts +++ b/src/ardrive.ts @@ -678,7 +678,14 @@ export class ArDrive extends ArDriveAnonymous { let uploadEntityResults: ArFSEntityData[] = []; let folderId: FolderID; - if (wrappedFolder.existingId) { + if (wrappedFolder.fileNameConflict) { + if (conflictResolution === skipOnConflicts) { + // Return empty result on skip + return { entityResults: [], feeResults: {} }; + } + // Otherwise throw an error, folder names cannot conflict with file names + throw new Error(errorMessage.entityNameExists); + } else if (wrappedFolder.existingId) { // Use existing parent folder ID for bulk upload folderId = wrappedFolder.existingId; } else { @@ -1003,7 +1010,7 @@ export class ArDrive extends ArDriveAnonymous { if (wrappedFolder.fileNameConflict) { if (conflictResolution === skipOnConflicts) { // Return empty result on skip - return { entityResults: uploadEntityResults, feeResults: uploadEntityFees }; + return { entityResults: [], feeResults: {} }; } // Otherwise throw an error, folder names cannot conflict with file names throw new Error(errorMessage.entityNameExists); From e2271bbdd37d0fd7800485ae2135589492c10750 Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Tue, 2 Nov 2021 10:31:50 -0500 Subject: [PATCH 20/31] refactor(conflicts): Use more descriptive vars PE-633 --- src/ardrive.ts | 6 +++--- src/arfs_file_wrapper.ts | 32 +++++++++++++++++--------------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/ardrive.ts b/src/ardrive.ts index 5604935e..56fe903f 100644 --- a/src/ardrive.ts +++ b/src/ardrive.ts @@ -678,7 +678,7 @@ export class ArDrive extends ArDriveAnonymous { let uploadEntityResults: ArFSEntityData[] = []; let folderId: FolderID; - if (wrappedFolder.fileNameConflict) { + if (wrappedFolder.existingFileAtDestConflict) { if (conflictResolution === skipOnConflicts) { // Return empty result on skip return { entityResults: [], feeResults: {} }; @@ -1007,7 +1007,7 @@ export class ArDrive extends ArDriveAnonymous { let uploadEntityResults: ArFSEntityData[] = []; let folderId: FolderID; - if (wrappedFolder.fileNameConflict) { + if (wrappedFolder.existingFileAtDestConflict) { if (conflictResolution === skipOnConflicts) { // Return empty result on skip return { entityResults: [], feeResults: {} }; @@ -1065,7 +1065,7 @@ export class ArDrive extends ArDriveAnonymous { continue; } - if (wrappedFile.folderNameConflict) { + if (wrappedFile.existingFolderAtDestConflict) { if (conflictResolution === skipOnConflicts) { // Continue loop, skip uploading this file continue; diff --git a/src/arfs_file_wrapper.ts b/src/arfs_file_wrapper.ts index 392f2ac8..1202e41a 100644 --- a/src/arfs_file_wrapper.ts +++ b/src/arfs_file_wrapper.ts @@ -62,7 +62,7 @@ export class ArFSFileToUpload { baseCosts?: BulkFileBaseCosts; existingId?: FileID; - folderNameConflict = false; + existingFolderAtDestConflict = false; hasSameLastModifiedDate = false; public gatherFileInfo(): FileInfo { @@ -109,7 +109,7 @@ export class ArFSFolderToUpload { baseCosts?: MetaDataBaseCosts; existingId?: FolderID; destinationName?: string; - fileNameConflict = false; + existingFileAtDestConflict = false; constructor(public readonly filePath: FilePath, public readonly fileStats: fs.Stats) { const entitiesInFolder = fs.readdirSync(this.filePath); @@ -143,23 +143,25 @@ export class ArFSFolderToUpload { for await (const file of this.files) { const baseFileName = file.getBaseFileName(); - const folderNameConflict = existingEntityNamesAndIds.folders.find( + const existingFolderAtDestConflict = existingEntityNamesAndIds.folders.find( ({ folderName }) => folderName === baseFileName ); - if (folderNameConflict) { + if (existingFolderAtDestConflict) { // Folder name cannot conflict with a file name - file.folderNameConflict = true; + file.existingFolderAtDestConflict = true; continue; } - const fileNameConflict = existingEntityNamesAndIds.files.find(({ fileName }) => fileName === baseFileName); + const existingFileAtDestConflict = existingEntityNamesAndIds.files.find( + ({ fileName }) => fileName === baseFileName + ); // Conflicting file name creates a REVISION by default - if (fileNameConflict) { - file.existingId = fileNameConflict.fileId; + if (existingFileAtDestConflict) { + file.existingId = existingFileAtDestConflict.fileId; - if (fileNameConflict.lastModifiedDate === file.lastModifiedDate) { + if (existingFileAtDestConflict.lastModifiedDate === file.lastModifiedDate) { // Check last modified date and set to true to resolve upsert conditional file.hasSameLastModifiedDate = true; } @@ -169,24 +171,24 @@ export class ArFSFolderToUpload { for await (const folder of this.folders) { const baseFolderName = folder.getBaseFileName(); - const fileNameConflict = existingEntityNamesAndIds.files.find( + const existingFileAtDestConflict = existingEntityNamesAndIds.files.find( ({ fileName }) => fileName === baseFolderName ); - if (fileNameConflict) { + if (existingFileAtDestConflict) { // Folder name cannot conflict with a file name - this.fileNameConflict = true; + this.existingFileAtDestConflict = true; continue; } - const folderNameConflict = existingEntityNamesAndIds.folders.find( + const existingFolderAtDestConflict = existingEntityNamesAndIds.folders.find( ({ folderName }) => folderName === baseFolderName ); // Conflicting folder name uses EXISTING folder by default - if (folderNameConflict) { + if (existingFolderAtDestConflict) { // Assigns existing id for later use - folder.existingId = folderNameConflict.folderId; + folder.existingId = existingFolderAtDestConflict.folderId; // Recurse into existing folder on folder name conflict await folder.checkAndAssignExistingNames(getExistingNamesFn); From 0afa240af870f86561075269f83554060afdcdb9 Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Tue, 2 Nov 2021 10:45:28 -0500 Subject: [PATCH 21/31] docs(conflicts): Add conflict resolution table PE-633 --- README.md | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index b1595a38..5ef8e8bb 100644 --- a/README.md +++ b/README.md @@ -690,7 +690,31 @@ NOTE: To upload to the root of a drive, specify its root folder ID as the parent ardrive drive-info -d "c7f87712-b54e-4491-bc96-1c5fa7b1da50" | jq -r '.rootFolderId' ``` -By default, the single `upload-file` command will use the upsert behavior. It will check the destination folder for a file with a conflicting name. If no conflicts are found, it will insert (upload) the file. +### Uploading a Folder with Files (Bulk Upload) + +Users can perform a bulk upload by using the upload-file command on a target folder. The command will reconstruct the folder hierarchy on local disk as ArFS folders on the permaweb and upload each file into their corresponding folders: + +```shell +ardrive upload-file --local-file-path /path/to/folder --parent-folder-id "9af694f6-4cfc-4eee-88a8-1b02704760c0" -w /path/to/wallet.json +``` + +### Name Conflict Resolution on Upload + +By default, the `upload-file` command will use the upsert behavior if existing entities are encountered in the destination folder tree that would cause naming conflicts. + +Expect the behaviors from the following table for each resolution setting: + +| Source Type | Conflict at Dest | `skip` | `replace` | `upsert` (default) | +| ----------- | ---------------- | ------ | --------- | ------------------ | +| File | None | Insert | Insert | Insert | +| File | Matching File | Skip | Update | skip | +| File | Different File | Skip | Update | update | +| File | Folder | Skip | Fail | Fail | +| Folder | None | Insert | Insert | Insert | +| Folder | File | Skip | Fail | Fail | +| Folder | Folder | Skip | Re-use | Re-use | + +The default upsert behavior will check the destination folder for a file with a conflicting name. If no conflicts are found, it will insert (upload) the file. In the case that there is a FILE to FILE name conflict found, it will only update it if necessary. To determine if an update is necessary, upsert will compare the last modified dates of conflicting file and the file being uploaded. When they are matching, the upload will be skipped. Otherwise the file will be updated as a new revision. @@ -700,23 +724,10 @@ To override the upsert behavior, use the `--replace` option to always make new r ardrive upload-file --replace --local-file-path /path/to/file.txt --parent-folder-id "9af694f6-4cfc-4eee-88a8-1b02704760c0" -w /path/to/wallet.json ``` -### Uploading a Folder with Files (Bulk Upload) - -Users can perform a bulk upload by using the upload-file command on a target folder. The command will reconstruct the folder hierarchy on local disk as ArFS folders on the permaweb and upload each file into their corresponding folders: - ```shell -ardrive upload-file --local-file-path /path/to/folder --parent-folder-id "9af694f6-4cfc-4eee-88a8-1b02704760c0" -w /path/to/wallet.json +ardrive upload-file --skip --local-file-path /path/to/file.txt --parent-folder-id "9af694f6-4cfc-4eee-88a8-1b02704760c0" -w /path/to/wallet.json ``` -This method of upload can be used to upload a large number of files and folders within the folder tree. If existing entities are encountered in the destination folder tree that would cause naming conflicts, expect the following default behaviors: - -- Folder names that conflict with a FILE name at the destination will cause an error to be thrown -- Folder names that conflict with a FOLDER name at the destination will use the existing folder ID (i.e. skip) rather than creating a new folder -- File names that conflict with a FOLDER name at the destination will cause an error to be thrown -- When the intended file name conflicts with a FILE name at the destination the file will be SKIPPED if the last modified date matches the file to upload. If they have different last modified dates, it will be uploaded as a new REVISION - -Similar to the single file upload, the above FILE to FILE name conflict resolution behavior can be modified. Use the `--replace` option to force new revisions on all conflicts within the bulk upload regardless of last modified date. Or use the `--skip` option to simply skip ALL name conflicts. - ### Fetching the Metadata of a File Entity Simply perform the file-info command to retrieve the metadata of a file: From 34f8ac3d26e2c44c95e8d0aa5058c13c4f54ead0 Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Tue, 2 Nov 2021 13:43:29 -0500 Subject: [PATCH 22/31] fix(skip): Apply skip/upsert logic to bulk estimation PE-633 --- src/CLICommand/parameters_helper.ts | 4 +- src/ardrive.ts | 70 ++++++++++++++++++++++------- 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/src/CLICommand/parameters_helper.ts b/src/CLICommand/parameters_helper.ts index e530d418..58dcdc9a 100644 --- a/src/CLICommand/parameters_helper.ts +++ b/src/CLICommand/parameters_helper.ts @@ -12,7 +12,7 @@ import { WalletFileParameter, PrivateParameter, ReplaceParameter, - UpsertParameter + SkipParameter } from '../parameter_declarations'; import { cliWalletDao } from '..'; import { DriveID, DriveKey } from '../types'; @@ -212,7 +212,7 @@ export class ParametersHelper { return replaceOnConflicts; } - if (this.getParameterValue(UpsertParameter)) { + if (this.getParameterValue(SkipParameter)) { return skipOnConflicts; } diff --git a/src/ardrive.ts b/src/ardrive.ts index 56fe903f..32c3e651 100644 --- a/src/ardrive.ts +++ b/src/ardrive.ts @@ -630,7 +630,7 @@ export class ArDrive extends ArDriveAnonymous { // Estimate and assert the cost of the entire bulk upload // This will assign the calculated base costs to each wrapped file and folder - const bulkEstimation = await this.estimateAndAssertCostOfBulkUpload(wrappedFolder); + const bulkEstimation = await this.estimateAndAssertCostOfBulkUpload(wrappedFolder, conflictResolution); // TODO: Add interactive confirmation of price estimation before uploading @@ -678,15 +678,20 @@ export class ArDrive extends ArDriveAnonymous { let uploadEntityResults: ArFSEntityData[] = []; let folderId: FolderID; + if ( + conflictResolution === skipOnConflicts && + (wrappedFolder.existingFileAtDestConflict || wrappedFolder.existingId) + ) { + // When conflict resolution is set to skip, return empty result if an + // existing folder is found or there is conflict with a file name + return { entityResults: [], feeResults: {} }; + } + if (wrappedFolder.existingFileAtDestConflict) { - if (conflictResolution === skipOnConflicts) { - // Return empty result on skip - return { entityResults: [], feeResults: {} }; - } - // Otherwise throw an error, folder names cannot conflict with file names + // Folder names cannot conflict with file names throw new Error(errorMessage.entityNameExists); } else if (wrappedFolder.existingId) { - // Use existing parent folder ID for bulk upload + // Re-use existing parent folder ID for bulk upload if it exists folderId = wrappedFolder.existingId; } else { // Create the parent folder @@ -942,7 +947,11 @@ export class ArDrive extends ArDriveAnonymous { // Estimate and assert the cost of the entire bulk upload // This will assign the calculated base costs to each wrapped file and folder - const bulkEstimation = await this.estimateAndAssertCostOfBulkUpload(wrappedFolder, driveKey); + const bulkEstimation = await this.estimateAndAssertCostOfBulkUpload( + wrappedFolder, + conflictResolution, + driveKey + ); // TODO: Add interactive confirmation of price estimation before uploading @@ -1007,16 +1016,20 @@ export class ArDrive extends ArDriveAnonymous { let uploadEntityResults: ArFSEntityData[] = []; let folderId: FolderID; + if ( + conflictResolution === skipOnConflicts && + (wrappedFolder.existingFileAtDestConflict || wrappedFolder.existingId) + ) { + // When conflict resolution is set to skip, return empty result if an + // existing folder is found or there is conflict with a file name + return { entityResults: [], feeResults: {} }; + } + if (wrappedFolder.existingFileAtDestConflict) { - if (conflictResolution === skipOnConflicts) { - // Return empty result on skip - return { entityResults: [], feeResults: {} }; - } - // Otherwise throw an error, folder names cannot conflict with file names + // Folder names cannot conflict with file names throw new Error(errorMessage.entityNameExists); } else if (wrappedFolder.existingId) { - // Use existing parent folder ID for bulk upload. - // This happens when the parent folder's name conflicts + // Re-use existing parent folder ID for bulk upload if it exists folderId = wrappedFolder.existingId; } else { // Create parent folder @@ -1332,14 +1345,24 @@ export class ArDrive extends ArDriveAnonymous { * */ async estimateAndAssertCostOfBulkUpload( folderToUpload: ArFSFolderToUpload, + conflictResolution: FileNameConflictResolution, driveKey?: DriveKey, isParentFolder = true ): Promise<{ totalPrice: Winston; totalFilePrice: Winston; communityWinstonTip: Winston }> { let totalPrice = 0; let totalFilePrice = 0; + if ( + conflictResolution === skipOnConflicts && + (folderToUpload.existingFileAtDestConflict || folderToUpload.existingId) + ) { + // When conflict resolution is set to skip, return empty estimation if an + // existing folder is found or there is conflict with a file name + return { totalPrice: '0', totalFilePrice: '0', communityWinstonTip: '0' }; + } + + // Don't estimate cost of folder metadata if using existing folder if (!folderToUpload.existingId) { - // Don't estimate cost of folder metadata if using existing folder const folderMetadataTrxData = await (async () => { const folderName = folderToUpload.destinationName ?? folderToUpload.getBaseFileName(); @@ -1360,6 +1383,14 @@ export class ArDrive extends ArDriveAnonymous { } for await (const file of folderToUpload.files) { + if ( + (conflictResolution === skipOnConflicts && (file.existingId || file.existingFolderAtDestConflict)) || + (conflictResolution === upsertOnConflicts && file.hasSameLastModifiedDate) + ) { + // File will skipped, don't estimate it; continue the loop + continue; + } + const fileSize = driveKey ? file.encryptedDataSize() : file.fileStats.size; const fileDataBaseReward = await this.priceEstimator.getBaseWinstonPriceForByteCount(fileSize); @@ -1384,7 +1415,12 @@ export class ArDrive extends ArDriveAnonymous { } for await (const folder of folderToUpload.folders) { - const childFolderResults = await this.estimateAndAssertCostOfBulkUpload(folder, driveKey, false); + const childFolderResults = await this.estimateAndAssertCostOfBulkUpload( + folder, + conflictResolution, + driveKey, + false + ); totalPrice += +childFolderResults.totalPrice; totalFilePrice += +childFolderResults.totalFilePrice; From dfa1bb82ae40d21c927d06b7a57b192ad1854b87 Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Tue, 2 Nov 2021 14:15:06 -0500 Subject: [PATCH 23/31] docs(conflicts): Capitalize words PE-633 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5ef8e8bb..4bec0311 100644 --- a/README.md +++ b/README.md @@ -707,8 +707,8 @@ Expect the behaviors from the following table for each resolution setting: | Source Type | Conflict at Dest | `skip` | `replace` | `upsert` (default) | | ----------- | ---------------- | ------ | --------- | ------------------ | | File | None | Insert | Insert | Insert | -| File | Matching File | Skip | Update | skip | -| File | Different File | Skip | Update | update | +| File | Matching File | Skip | Update | Skip | +| File | Different File | Skip | Update | Update | | File | Folder | Skip | Fail | Fail | | Folder | None | Insert | Insert | Insert | | Folder | File | Skip | Fail | Fail | From 5daf3859bde0b0fa152bf20816797d0289f823d7 Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Tue, 2 Nov 2021 14:33:43 -0500 Subject: [PATCH 24/31] fix(bulk upload): Don't upload .DS_Store PE-633 --- src/arfs_file_wrapper.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/arfs_file_wrapper.ts b/src/arfs_file_wrapper.ts index 1202e41a..d73d6c80 100644 --- a/src/arfs_file_wrapper.ts +++ b/src/arfs_file_wrapper.ts @@ -125,7 +125,9 @@ export class ArFSFolderToUpload { } else { // Child is a file, build a new file const childFile = new ArFSFileToUpload(absoluteEntityPath, entityStats); - this.files.push(childFile); + if (childFile.getBaseFileName() !== '.DS_Store') { + this.files.push(childFile); + } } } } From eaa223423c2e89bb73843e057d0a4678f039d18d Mon Sep 17 00:00:00 2001 From: Jordan Schau <412028+jordanschau@users.noreply.github.com> Date: Fri, 5 Nov 2021 15:21:31 -0700 Subject: [PATCH 25/31] Updated ArFS Link Link was broken and I believe the new link is correct. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 840d8c9f..2faada1d 100644 --- a/README.md +++ b/README.md @@ -880,7 +880,7 @@ ardrive --help [ardrive]: https://ardrive.io [arweave]: https://ardrive.io/what-is-arweave/ [ardrive-github]: https://github.com/ardriveapp/ -[arfs]: https://ardrive.atlassian.net/l/c/yDcGDbUm +[arfs]: https://ardrive.atlassian.net/wiki/spaces/help/pages/278495281/Arweave+File+System [ardrive-web-app]: https://app.ardrive.io [ardrive-core]: https://github.com/ardriveapp/ardrive-core-js [yarn-install]: https://yarnpkg.com/getting-started/install From 910195a653311f7971ae8138cc36fef63151c841 Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Tue, 9 Nov 2021 12:14:51 -0600 Subject: [PATCH 26/31] refactor(skip): Re-use existing folder on --skip PE-638 --- README.md | 2 +- src/ardrive.ts | 43 +++++++++++-------------------------------- 2 files changed, 12 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 419977c1..2cfb5843 100644 --- a/README.md +++ b/README.md @@ -716,7 +716,7 @@ Expect the behaviors from the following table for each resolution setting: | File | Folder | Skip | Fail | Fail | | Folder | None | Insert | Insert | Insert | | Folder | File | Skip | Fail | Fail | -| Folder | Folder | Skip | Re-use | Re-use | +| Folder | Folder | Re-use | Re-use | Re-use | The default upsert behavior will check the destination folder for a file with a conflicting name. If no conflicts are found, it will insert (upload) the file. diff --git a/src/ardrive.ts b/src/ardrive.ts index 5244ef4b..274a4759 100644 --- a/src/ardrive.ts +++ b/src/ardrive.ts @@ -680,18 +680,10 @@ export class ArDrive extends ArDriveAnonymous { let uploadEntityResults: ArFSEntityData[] = []; let folderId: FolderID; - if ( - conflictResolution === skipOnConflicts && - (wrappedFolder.existingFileAtDestConflict || wrappedFolder.existingId) - ) { - // When conflict resolution is set to skip, return empty result if an - // existing folder is found or there is conflict with a file name - return { entityResults: [], feeResults: {} }; - } - if (wrappedFolder.existingFileAtDestConflict) { // Folder names cannot conflict with file names - throw new Error(errorMessage.entityNameExists); + // Return an empty result to continue other parts of upload + return { entityResults: [], feeResults: {} }; } else if (wrappedFolder.existingId) { // Re-use existing parent folder ID for bulk upload if it exists folderId = wrappedFolder.existingId; @@ -734,11 +726,14 @@ export class ArDrive extends ArDriveAnonymous { // Conflict resolution is set to skip and there is an existing file (conflictResolution === skipOnConflicts && wrappedFile.existingId) || // Conflict resolution is set to upsert and an existing file has the same last modified date - (conflictResolution === upsertOnConflicts && wrappedFile.hasSameLastModifiedDate) + (conflictResolution === upsertOnConflicts && wrappedFile.hasSameLastModifiedDate) || + // File names cannot conflict with folder names + wrappedFile.existingFolderAtDestConflict ) { // Continue loop, don't upload this file continue; } + const fileDataRewardSettings = { reward: wrappedFile.getBaseCosts().fileDataBaseReward, feeMultiple: this.feeMultiple @@ -1018,18 +1013,10 @@ export class ArDrive extends ArDriveAnonymous { let uploadEntityResults: ArFSEntityData[] = []; let folderId: FolderID; - if ( - conflictResolution === skipOnConflicts && - (wrappedFolder.existingFileAtDestConflict || wrappedFolder.existingId) - ) { - // When conflict resolution is set to skip, return empty result if an - // existing folder is found or there is conflict with a file name - return { entityResults: [], feeResults: {} }; - } - if (wrappedFolder.existingFileAtDestConflict) { // Folder names cannot conflict with file names - throw new Error(errorMessage.entityNameExists); + // Return an empty result to continue other parts of upload + return { entityResults: [], feeResults: {} }; } else if (wrappedFolder.existingId) { // Re-use existing parent folder ID for bulk upload if it exists folderId = wrappedFolder.existingId; @@ -1074,22 +1061,14 @@ export class ArDrive extends ArDriveAnonymous { // Conflict resolution is set to skip and there is an existing file (conflictResolution === skipOnConflicts && wrappedFile.existingId) || // Conflict resolution is set to upsert and an existing file has the same last modified date - (conflictResolution === upsertOnConflicts && wrappedFile.hasSameLastModifiedDate) + (conflictResolution === upsertOnConflicts && wrappedFile.hasSameLastModifiedDate) || + // File names cannot conflict with folder names + wrappedFile.existingFolderAtDestConflict ) { // Continue loop, don't upload this file continue; } - if (wrappedFile.existingFolderAtDestConflict) { - if (conflictResolution === skipOnConflicts) { - // Continue loop, skip uploading this file - continue; - } - - // Otherwise throw an error, file names cannot conflict with folder names - throw new Error(errorMessage.entityNameExists); - } - const fileDataRewardSettings = { reward: wrappedFile.getBaseCosts().fileDataBaseReward, feeMultiple: this.feeMultiple From dec43610de4a626cb9381cccf8797f4f48bfdf9b Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Tue, 9 Nov 2021 14:44:12 -0600 Subject: [PATCH 27/31] refactor(skip): Update bulk estimation with new behavior PE-638 --- src/ardrive.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/ardrive.ts b/src/ardrive.ts index 274a4759..ba16bb94 100644 --- a/src/ardrive.ts +++ b/src/ardrive.ts @@ -1333,12 +1333,8 @@ export class ArDrive extends ArDriveAnonymous { let totalPrice = 0; let totalFilePrice = 0; - if ( - conflictResolution === skipOnConflicts && - (folderToUpload.existingFileAtDestConflict || folderToUpload.existingId) - ) { - // When conflict resolution is set to skip, return empty estimation if an - // existing folder is found or there is conflict with a file name + if (folderToUpload.existingFileAtDestConflict) { + // Return an empty estimation, folders CANNOT overwrite files return { totalPrice: '0', totalFilePrice: '0', communityWinstonTip: '0' }; } @@ -1365,8 +1361,9 @@ export class ArDrive extends ArDriveAnonymous { for await (const file of folderToUpload.files) { if ( - (conflictResolution === skipOnConflicts && (file.existingId || file.existingFolderAtDestConflict)) || - (conflictResolution === upsertOnConflicts && file.hasSameLastModifiedDate) + (conflictResolution === skipOnConflicts && file.existingId) || + (conflictResolution === upsertOnConflicts && file.hasSameLastModifiedDate) || + file.existingFolderAtDestConflict ) { // File will skipped, don't estimate it; continue the loop continue; From 938de6bbc653d347bba55b0acc0db2d37c36f9fa Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Wed, 10 Nov 2021 10:17:32 -0600 Subject: [PATCH 28/31] fix(tip tags): Use consistent tags with web app PE-675 --- src/ardrive.ts | 1 + src/commands/send_ar.ts | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/ardrive.ts b/src/ardrive.ts index a5a5476a..df7344a2 100644 --- a/src/ardrive.ts +++ b/src/ardrive.ts @@ -219,6 +219,7 @@ export class ArDrive extends ArDriveAnonymous { return [ { name: 'App-Name', value: this.appName }, { name: 'App-Version', value: this.appVersion }, + { name: 'Type', value: 'fee' }, { name: 'Tip-Type', value: tipType } ]; } diff --git a/src/commands/send_ar.ts b/src/commands/send_ar.ts index dbd120ea..a27a4f20 100644 --- a/src/commands/send_ar.ts +++ b/src/commands/send_ar.ts @@ -1,4 +1,4 @@ -import { cliWalletDao } from '..'; +import { cliWalletDao, CLI_APP_NAME, CLI_APP_VERSION } from '..'; import { ArweaveAddress } from '../arweave_address'; import { CLICommand } from '../CLICommand'; import { ParametersHelper } from '../CLICommand'; @@ -34,8 +34,8 @@ new CLICommand({ rewardSetting, options.dryRun, [ - { name: 'App-Name', value: 'ArDrive-CLI' }, - { name: 'App-Version', value: '2.0' }, + { name: 'App-Name', value: CLI_APP_NAME }, + { name: 'App-Version', value: CLI_APP_VERSION }, { name: 'Type', value: 'transfer' } ], true From fd46d8f4d7a74fa1b578dd8e59334ed65f6d40b1 Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Wed, 10 Nov 2021 10:18:16 -0600 Subject: [PATCH 29/31] chore(version): Bump to 1.0.3 PE-675 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9481ab6f..416f7ba5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ardrive-cli", - "version": "1.0.2", + "version": "1.0.3", "description": "The ArDrive Command Line Interface (CLI is a Node.js application for terminal-based ArDrive workflows. It also offers utility operations for securely interacting with Arweave wallets and inspecting various Arweave blockchain conditions.", "main": "./lib/index.js", "bin": { From 26f70fd95cbadc83db9d4007304cd6d4702aed9d Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Wed, 10 Nov 2021 10:26:12 -0600 Subject: [PATCH 30/31] test(tip tags): Adjust unit tests PE-675 --- src/utils/ardrive.test.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/utils/ardrive.test.ts b/src/utils/ardrive.test.ts index 864dcd11..63dea349 100644 --- a/src/utils/ardrive.test.ts +++ b/src/utils/ardrive.test.ts @@ -97,10 +97,14 @@ describe('ArDrive class', () => { { name: 'App-Version', value: '1.0' } ]; const inputsAndExpectedOutputs = [ - [undefined, [...baseTags, { name: 'Tip-Type', value: 'data upload' }]], - ['data upload', [...baseTags, { name: 'Tip-Type', value: 'data upload' }]] + [undefined, [...baseTags, { name: 'Type', value: 'fee' }, { name: 'Tip-Type', value: 'data upload' }]], + [ + 'data upload', + [...baseTags, { name: 'Type', value: 'fee' }, { name: 'Tip-Type', value: 'data upload' }] + ] ]; inputsAndExpectedOutputs.forEach(([input, expectedOutput]) => { + console.log(JSON.stringify(expectedOutput, null, 4)); expect(arDrive.getTipTags(input as TipType)).to.deep.equal(expectedOutput); }); }); From b049796c603806f27af1eaef56682b2549313e6c Mon Sep 17 00:00:00 2001 From: Derek Sonnenberg Date: Wed, 10 Nov 2021 16:06:06 -0600 Subject: [PATCH 31/31] fix(arfs link): Use more permanent link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 20a04a05..b6b28999 100644 --- a/README.md +++ b/README.md @@ -903,7 +903,7 @@ ardrive --help [ardrive]: https://ardrive.io [arweave]: https://ardrive.io/what-is-arweave/ [ardrive-github]: https://github.com/ardriveapp/ -[arfs]: https://ardrive.atlassian.net/wiki/spaces/help/pages/278495281/Arweave+File+System +[arfs]: https://ardrive.atlassian.net/l/c/m6P1vJDo [ardrive-web-app]: https://app.ardrive.io [ardrive-core]: https://github.com/ardriveapp/ardrive-core-js [yarn-install]: https://yarnpkg.com/getting-started/install