From c76e6fbe9d8e14a2843a91a35c67e9cda65091c6 Mon Sep 17 00:00:00 2001 From: jdaev Date: Tue, 26 Jul 2022 09:54:03 -0400 Subject: [PATCH 01/53] feat: add download function PE-1904 --- src/utils/download.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 src/utils/download.ts diff --git a/src/utils/download.ts b/src/utils/download.ts new file mode 100644 index 00000000..bc3fdb82 --- /dev/null +++ b/src/utils/download.ts @@ -0,0 +1,24 @@ +import { get } from 'http'; +import { createWriteStream } from 'fs'; + +export async function download(url: string, dest: string): Promise { + const file = createWriteStream(dest); + return new Promise((resolve, reject) => { + let responseSent = false; // flag to make sure that response is sent only once. + get(url, (response) => { + response.pipe(file); + file.on('finish', () => { + file.close(); + () => { + if (responseSent) return; + responseSent = true; + resolve(); + }; + }); + }).on('error', (err) => { + if (responseSent) return; + responseSent = true; + reject(err); + }); + }); +} From 62377ff9ccf5abc4c440fff86449a563fe7d3e29 Mon Sep 17 00:00:00 2001 From: jdaev Date: Tue, 26 Jul 2022 09:55:16 -0400 Subject: [PATCH 02/53] refactor: add remote path parameter PE-1904 --- src/commands/upload_file.ts | 23 ++++++++++++++++++++--- src/parameter_declarations.ts | 21 ++++++++++++++++++++- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index 797a0bc2..579412a8 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -13,7 +13,8 @@ import { LocalPathParameter, LocalCSVParameter, GatewayParameter, - CustomContentTypeParameter + CustomContentTypeParameter, + RemotePathParameter } from '../parameter_declarations'; import { fileAndFolderUploadConflictPrompts } from '../prompts'; import { ERROR_EXIT_CODE, SUCCESS_EXIT_CODE } from '../CLICommand/error_codes'; @@ -48,7 +49,6 @@ function getFilesFromCSV(parameters: ParametersHelper): UploadPathParameter[] | if (!localCSVFile) { return undefined; } - const localCSVFileData = fs.readFileSync(localCSVFile).toString().trim(); const COLUMN_SEPARATOR = ','; const ROW_SEPARATOR = '\n'; @@ -118,6 +118,21 @@ function getSingleFile(parameters: ParametersHelper, parentFolderId: FolderID): return [singleParameter]; } +// function getRemoteFile(parameters: ParametersHelper, parentFolderId: FolderID): UploadPathParameter[] { +// const remoteFilePath = parameters.getParameterValue(RemotePathParameter); + +// const customContentType = parameters.getParameterValue(CustomContentTypeParameter); + +// const wrappedEntity = wrapFileOrFolder(localFilePath, customContentType); +// const singleParameter = { +// parentFolderId: parentFolderId, +// wrappedEntity, +// destinationFileName: parameters.getParameterValue(DestinationFileNameParameter) +// }; + +// return [singleParameter]; +// } + new CLICommand({ name: 'upload-file', parameters: [ @@ -136,7 +151,8 @@ new CLICommand({ LocalFilePathParameter_DEPRECATED, LocalFilesParameter_DEPRECATED, BoostParameter, - GatewayParameter + GatewayParameter, + RemotePathParameter ], action: new CLIAction(async function action(options) { const parameters = new ParametersHelper(options); @@ -148,6 +164,7 @@ new CLICommand({ return filesFromCSV; } + //const filesFromRemote = getFilesFromRemotePath(parameters); // Determine list of files to upload and destinations from parameter list // First check the multi-file input case const parentFolderId: FolderID = parameters.getRequiredParameterValue(ParentFolderIdParameter, EID); diff --git a/src/parameter_declarations.ts b/src/parameter_declarations.ts index ef9be2e3..6b043a11 100644 --- a/src/parameter_declarations.ts +++ b/src/parameter_declarations.ts @@ -41,6 +41,7 @@ export const LocalCSVParameter = 'localCsv'; export const WithKeysParameter = 'withKeys'; export const GatewayParameter = 'gateway'; export const CustomContentTypeParameter = 'contentType'; +export const RemotePathParameter = 'remotePath'; // Aggregates for convenience export const WalletTypeParameters = [WalletFileParameter, SeedPhraseParameter]; @@ -81,7 +82,8 @@ export const AllParameters = [ TxFilePathParameter, UnsafeDrivePasswordParameter, WalletFileParameter, - WithKeysParameter + WithKeysParameter, + RemotePathParameter ] as const; export type ParameterName = typeof AllParameters[number]; @@ -466,3 +468,20 @@ Parameter.declare({ description: '(OPTIONAL) Provide a custom content type to all files within the upload to be used by the gateway to display the content' }); + +Parameter.declare({ + name: RemotePathParameter, + aliases: ['--remote-path'], + description: `the remote path for the file that will be uploaded +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-file-path +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-files +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-paths +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-path +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-csv`, + forbiddenConjunctionParameters: [ + LocalFilePathParameter_DEPRECATED, + LocalPathsParameter, + LocalCSVParameter, + LocalPathParameter + ] +}); From d17d8646ee73df30b172962efb78734cddd13ab4 Mon Sep 17 00:00:00 2001 From: jdaev Date: Tue, 26 Jul 2022 11:37:49 -0400 Subject: [PATCH 03/53] feat: creat temp folder util PE-1904 --- src/utils/temp.ts | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 src/utils/temp.ts diff --git a/src/utils/temp.ts b/src/utils/temp.ts new file mode 100644 index 00000000..aa9e1b49 --- /dev/null +++ b/src/utils/temp.ts @@ -0,0 +1,25 @@ +import * as fs from 'fs'; +import * as os from 'os'; +import path from 'path'; + +function platformTempFolder(): string { + const tempBaseFolder = process.env['XDG_RUNTIME_DIR'] ?? os.homedir(); + return os.platform() === 'win32' + ? path.join(tempBaseFolder, 'ardrive-temp') + : path.join(tempBaseFolder, '.ardrive', 'temp'); +} + +export async function getTempFolder(): Promise { + const tempFolderPath = await platformTempFolder(); + if (fs.existsSync(tempFolderPath)) { + return tempFolderPath; + } + const folder = await fs.promises.mkdir(`${tempFolderPath}`, { recursive: true }).then((result) => { + if (!result) { + throw new Error('Could not create ardrive-cli temp!'); + } + + return tempFolderPath; + }); + return folder; +} From 8627de0d1ccc08becddbdd79756c5badd8dc995f Mon Sep 17 00:00:00 2001 From: jdaev Date: Tue, 26 Jul 2022 18:18:55 -0400 Subject: [PATCH 04/53] refactor: hook up download, cache and command PE-1904 --- src/commands/upload_file.ts | 45 ++++++++++++++++++++++-------- src/utils/download.ts | 55 +++++++++++++++++++++++-------------- 2 files changed, 68 insertions(+), 32 deletions(-) diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index 579412a8..002abc1b 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -32,6 +32,8 @@ import { import { cliArDriveFactory } from '..'; import * as fs from 'fs'; import { getArweaveFromURL } from '../utils/get_arweave_for_url'; +import { getTempFolder } from '../utils/temp'; +import { download } from '../utils/download'; interface UploadPathParameter { parentFolderId: FolderID; @@ -118,20 +120,34 @@ function getSingleFile(parameters: ParametersHelper, parentFolderId: FolderID): return [singleParameter]; } -// function getRemoteFile(parameters: ParametersHelper, parentFolderId: FolderID): UploadPathParameter[] { -// const remoteFilePath = parameters.getParameterValue(RemotePathParameter); +async function getRemoteFile( + parameters: ParametersHelper, + parentFolderId: FolderID +): Promise { + const remoteFilePath = parameters.getParameterValue(RemotePathParameter); -// const customContentType = parameters.getParameterValue(CustomContentTypeParameter); + if (!remoteFilePath) { + return undefined; + } + + const tempFolder = await getTempFolder(); -// const wrappedEntity = wrapFileOrFolder(localFilePath, customContentType); -// const singleParameter = { -// parentFolderId: parentFolderId, -// wrappedEntity, -// destinationFileName: parameters.getParameterValue(DestinationFileNameParameter) -// }; + const localFilePath = await download(remoteFilePath, tempFolder); -// return [singleParameter]; -// } + if (!localFilePath) { + return undefined; + } + const customContentType = parameters.getParameterValue(CustomContentTypeParameter); + + const wrappedEntity = wrapFileOrFolder(localFilePath, customContentType); + const singleParameter = { + parentFolderId: parentFolderId, + wrappedEntity, + destinationFileName: parameters.getParameterValue(DestinationFileNameParameter) + }; + + return [singleParameter]; +} new CLICommand({ name: 'upload-file', @@ -164,7 +180,6 @@ new CLICommand({ return filesFromCSV; } - //const filesFromRemote = getFilesFromRemotePath(parameters); // Determine list of files to upload and destinations from parameter list // First check the multi-file input case const parentFolderId: FolderID = parameters.getRequiredParameterValue(ParentFolderIdParameter, EID); @@ -173,6 +188,12 @@ new CLICommand({ return fileList; } + const filesFromRemote = await getRemoteFile(parameters, parentFolderId); + + if (filesFromRemote) { + return filesFromRemote; + } + // If neither the multi-file input case or csv case produced files, try the single file case (deprecated) return getSingleFile(parameters, parentFolderId); })(); diff --git a/src/utils/download.ts b/src/utils/download.ts index bc3fdb82..f62b8e26 100644 --- a/src/utils/download.ts +++ b/src/utils/download.ts @@ -1,24 +1,39 @@ -import { get } from 'http'; -import { createWriteStream } from 'fs'; - -export async function download(url: string, dest: string): Promise { - const file = createWriteStream(dest); - return new Promise((resolve, reject) => { - let responseSent = false; // flag to make sure that response is sent only once. - get(url, (response) => { +import * as http from 'http'; +import * as https from 'https'; +import * as fs from 'fs'; + +/** + * Downloads file from remote HTTP[S] host and puts its contents to the + * specified location. + */ +export async function download(url: string, filePath: string): Promise { + const proto = !url.charAt(4).localeCompare('s') ? https : http; + + return new Promise((resolve, reject) => { + const file = fs.createWriteStream(filePath); + + const request = proto.get(url, (response) => { + if (response.statusCode !== 200) { + fs.unlink(filePath, () => { + reject(new Error(`Failed to get '${url}' (${response.statusCode})`)); + }); + return; + } + response.pipe(file); - file.on('finish', () => { - file.close(); - () => { - if (responseSent) return; - responseSent = true; - resolve(); - }; - }); - }).on('error', (err) => { - if (responseSent) return; - responseSent = true; - reject(err); }); + + // The destination stream is ended by the time it's called + file.on('finish', () => resolve(file.path.toString())); + + request.on('error', (err: unknown) => { + fs.unlink(filePath, () => reject(err)); + }); + + file.on('error', (err: unknown) => { + fs.unlink(filePath, () => reject(err)); + }); + + request.end(); }); } From 32f4b458fad81dc461221d417fc9082e5542d21e Mon Sep 17 00:00:00 2001 From: jdaev Date: Wed, 27 Jul 2022 10:29:36 -0400 Subject: [PATCH 05/53] refactor: follow remote path redirects PE-1904 --- src/utils/download.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/utils/download.ts b/src/utils/download.ts index f62b8e26..5ceb54b1 100644 --- a/src/utils/download.ts +++ b/src/utils/download.ts @@ -8,18 +8,25 @@ import * as fs from 'fs'; */ export async function download(url: string, filePath: string): Promise { const proto = !url.charAt(4).localeCompare('s') ? https : http; - + console.log(filePath); return new Promise((resolve, reject) => { - const file = fs.createWriteStream(filePath); + const file = fs.createWriteStream(filePath, { flags: 'wx' }); const request = proto.get(url, (response) => { - if (response.statusCode !== 200) { + if (response.statusCode === 302 || response.statusCode === 301) { + //Recursively follow redirects, only a 200 will resolve. + if (response.headers.location) { + download(response.headers.location, filePath).then((filePath: string | undefined) => + resolve(filePath) + ); + } + } else if (response.statusCode !== 200) { fs.unlink(filePath, () => { reject(new Error(`Failed to get '${url}' (${response.statusCode})`)); }); return; } - + console.log(file.path); response.pipe(file); }); From abd4aaff576ccdad7dc16b6f8bdb37ee542fecbf Mon Sep 17 00:00:00 2001 From: jdaev Date: Wed, 27 Jul 2022 12:00:39 -0400 Subject: [PATCH 06/53] refactor: derive file name from url PE-1904 --- src/utils/download.ts | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/utils/download.ts b/src/utils/download.ts index 5ceb54b1..d575f1db 100644 --- a/src/utils/download.ts +++ b/src/utils/download.ts @@ -1,6 +1,8 @@ import * as http from 'http'; import * as https from 'https'; import * as fs from 'fs'; +import path from 'path'; +import { reject } from 'lodash'; /** * Downloads file from remote HTTP[S] host and puts its contents to the @@ -8,9 +10,16 @@ import * as fs from 'fs'; */ export async function download(url: string, filePath: string): Promise { const proto = !url.charAt(4).localeCompare('s') ? https : http; - console.log(filePath); + const fileName = url.split('/').pop(); + if (!fileName) { + reject(undefined); + return; + } + console.log(url); + const pathToFile = path.join(filePath, fileName); + console.log(pathToFile); return new Promise((resolve, reject) => { - const file = fs.createWriteStream(filePath, { flags: 'wx' }); + const file = fs.createWriteStream(pathToFile, { flags: 'wx' }); const request = proto.get(url, (response) => { if (response.statusCode === 302 || response.statusCode === 301) { @@ -20,25 +29,24 @@ export async function download(url: string, filePath: string): Promise { + fs.unlink(pathToFile, () => { reject(new Error(`Failed to get '${url}' (${response.statusCode})`)); }); return; } console.log(file.path); - response.pipe(file); }); - // The destination stream is ended by the time it's called file.on('finish', () => resolve(file.path.toString())); request.on('error', (err: unknown) => { - fs.unlink(filePath, () => reject(err)); + fs.unlink(pathToFile, () => reject(err)); }); file.on('error', (err: unknown) => { - fs.unlink(filePath, () => reject(err)); + fs.unlink(pathToFile, () => reject(err)); }); request.end(); From bdde9834b0866723145306a0b618e2f83283dd6e Mon Sep 17 00:00:00 2001 From: jdaev Date: Wed, 27 Jul 2022 15:18:28 -0400 Subject: [PATCH 07/53] fix: fix file save errors PE-1904 --- src/commands/upload_file.ts | 4 ++-- src/utils/download.ts | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index 002abc1b..68c605fc 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -134,6 +134,7 @@ async function getRemoteFile( const localFilePath = await download(remoteFilePath, tempFolder); + console.log(localFilePath); if (!localFilePath) { return undefined; } @@ -187,9 +188,8 @@ new CLICommand({ if (fileList) { return fileList; } - const filesFromRemote = await getRemoteFile(parameters, parentFolderId); - + console.log(filesFromRemote); if (filesFromRemote) { return filesFromRemote; } diff --git a/src/utils/download.ts b/src/utils/download.ts index d575f1db..ecb59dc0 100644 --- a/src/utils/download.ts +++ b/src/utils/download.ts @@ -15,11 +15,9 @@ export async function download(url: string, filePath: string): Promise { - const file = fs.createWriteStream(pathToFile, { flags: 'wx' }); + const file = fs.createWriteStream(pathToFile); const request = proto.get(url, (response) => { if (response.statusCode === 302 || response.statusCode === 301) { @@ -36,10 +34,12 @@ export async function download(url: string, filePath: string): Promise resolve(file.path.toString())); + + file.on('finish', () => { + resolve(file.path.toString()); + }); request.on('error', (err: unknown) => { fs.unlink(pathToFile, () => reject(err)); From 09437cc444bb889446902f7a69cd59d0953ae95d Mon Sep 17 00:00:00 2001 From: Javed Hussein Date: Wed, 27 Jul 2022 15:37:48 -0400 Subject: [PATCH 08/53] fix: remove wx flag PE-1904 --- src/utils/download.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/download.ts b/src/utils/download.ts index d575f1db..31fbb307 100644 --- a/src/utils/download.ts +++ b/src/utils/download.ts @@ -19,7 +19,7 @@ export async function download(url: string, filePath: string): Promise { - const file = fs.createWriteStream(pathToFile, { flags: 'wx' }); + const file = fs.createWriteStream(pathToFile); const request = proto.get(url, (response) => { if (response.statusCode === 302 || response.statusCode === 301) { From 07a3e9191e108f2dcdad4ad4692b4e8ebae14ed6 Mon Sep 17 00:00:00 2001 From: Karl Prieb Date: Wed, 27 Jul 2022 17:14:11 -0300 Subject: [PATCH 09/53] fix(download): use axios --- src/commands/upload_file.ts | 2 +- src/utils/download.ts | 44 +++++-------------------------------- src/utils/temp.ts | 14 +++++------- 3 files changed, 11 insertions(+), 49 deletions(-) diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index 002abc1b..f869beea 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -130,7 +130,7 @@ async function getRemoteFile( return undefined; } - const tempFolder = await getTempFolder(); + const tempFolder = getTempFolder(); const localFilePath = await download(remoteFilePath, tempFolder); diff --git a/src/utils/download.ts b/src/utils/download.ts index 31fbb307..50b84cc3 100644 --- a/src/utils/download.ts +++ b/src/utils/download.ts @@ -1,54 +1,20 @@ -import * as http from 'http'; -import * as https from 'https'; import * as fs from 'fs'; import path from 'path'; -import { reject } from 'lodash'; +import axios, { AxiosResponse } from 'axios'; /** * Downloads file from remote HTTP[S] host and puts its contents to the * specified location. */ export async function download(url: string, filePath: string): Promise { - const proto = !url.charAt(4).localeCompare('s') ? https : http; const fileName = url.split('/').pop(); if (!fileName) { - reject(undefined); return; } - console.log(url); + const response: AxiosResponse = await axios.get(url, { responseType: 'arraybuffer' }); + console.log(response.headers['content-type']); const pathToFile = path.join(filePath, fileName); - console.log(pathToFile); - return new Promise((resolve, reject) => { - const file = fs.createWriteStream(pathToFile); - const request = proto.get(url, (response) => { - if (response.statusCode === 302 || response.statusCode === 301) { - //Recursively follow redirects, only a 200 will resolve. - if (response.headers.location) { - download(response.headers.location, filePath).then((filePath: string | undefined) => - resolve(filePath) - ); - } - return; - } else if (response.statusCode !== 200) { - fs.unlink(pathToFile, () => { - reject(new Error(`Failed to get '${url}' (${response.statusCode})`)); - }); - return; - } - console.log(file.path); - }); - // The destination stream is ended by the time it's called - file.on('finish', () => resolve(file.path.toString())); - - request.on('error', (err: unknown) => { - fs.unlink(pathToFile, () => reject(err)); - }); - - file.on('error', (err: unknown) => { - fs.unlink(pathToFile, () => reject(err)); - }); - - request.end(); - }); + fs.writeFileSync(pathToFile, response.data); + return pathToFile; } diff --git a/src/utils/temp.ts b/src/utils/temp.ts index aa9e1b49..5309ee34 100644 --- a/src/utils/temp.ts +++ b/src/utils/temp.ts @@ -9,17 +9,13 @@ function platformTempFolder(): string { : path.join(tempBaseFolder, '.ardrive', 'temp'); } -export async function getTempFolder(): Promise { - const tempFolderPath = await platformTempFolder(); +export function getTempFolder(): string { + const tempFolderPath = platformTempFolder(); if (fs.existsSync(tempFolderPath)) { return tempFolderPath; } - const folder = await fs.promises.mkdir(`${tempFolderPath}`, { recursive: true }).then((result) => { - if (!result) { - throw new Error('Could not create ardrive-cli temp!'); - } - return tempFolderPath; - }); - return folder; + fs.mkdirSync(tempFolderPath); + + return tempFolderPath; } From aa1e1d45f0189bfd99f5bd0c34bbbfc2c56b4630 Mon Sep 17 00:00:00 2001 From: Javed Hussein Date: Fri, 29 Jul 2022 10:04:26 -0400 Subject: [PATCH 10/53] refactor: add comments and errors PE-1904 --- src/utils/download.ts | 18 ++++++++++++------ src/utils/temp.ts | 4 +++- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/utils/download.ts b/src/utils/download.ts index 50b84cc3..1bcdf40d 100644 --- a/src/utils/download.ts +++ b/src/utils/download.ts @@ -5,16 +5,22 @@ import axios, { AxiosResponse } from 'axios'; /** * Downloads file from remote HTTP[S] host and puts its contents to the * specified location. + * @param url URL of the file to download. + * @param destinationPath Path to the destination file. */ export async function download(url: string, filePath: string): Promise { const fileName = url.split('/').pop(); if (!fileName) { - return; + throw new Error('Invalid remote path. No file name found.'); } - const response: AxiosResponse = await axios.get(url, { responseType: 'arraybuffer' }); - console.log(response.headers['content-type']); - const pathToFile = path.join(filePath, fileName); + try { + const response: AxiosResponse = await axios.get(url, { responseType: 'arraybuffer' }); + console.log(response.headers['content-type']); + const pathToFile = path.join(filePath, fileName); - fs.writeFileSync(pathToFile, response.data); - return pathToFile; + fs.writeFileSync(pathToFile, response.data); + return pathToFile; + } catch (error) { + throw new Error(`Failed to download file from remote path ${url}: ${error.message}`); + } } diff --git a/src/utils/temp.ts b/src/utils/temp.ts index 5309ee34..6ccec845 100644 --- a/src/utils/temp.ts +++ b/src/utils/temp.ts @@ -8,7 +8,9 @@ function platformTempFolder(): string { ? path.join(tempBaseFolder, 'ardrive-temp') : path.join(tempBaseFolder, '.ardrive', 'temp'); } - +/** + * Gets a folder path for storing temporary files. + */ export function getTempFolder(): string { const tempFolderPath = platformTempFolder(); if (fs.existsSync(tempFolderPath)) { From eca8aabe0d4b879ad89b9921817a1c7565e27a69 Mon Sep 17 00:00:00 2001 From: jdaev Date: Fri, 29 Jul 2022 10:55:32 -0400 Subject: [PATCH 11/53] test: add temp folder test PE-1904 --- src/utils/temp.test.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 src/utils/temp.test.ts diff --git a/src/utils/temp.test.ts b/src/utils/temp.test.ts new file mode 100644 index 00000000..b0f4dab6 --- /dev/null +++ b/src/utils/temp.test.ts @@ -0,0 +1,16 @@ +import { expect } from 'chai'; +import { getTempFolder } from './temp'; +import * as fs from 'fs'; + +describe('temp folder test', () => { + it('temp returns a folder that exists', () => { + const tempFolderPath = getTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(true); + }); + + it('temp path returned contains the correct subfolders', () => { + const tempFolderPath = getTempFolder(); + expect(tempFolderPath).to.contains('ardrive'); + expect(tempFolderPath).to.contains('temp'); + }); +}); From d83d060cb46bca14076748de9912c23c834e0b72 Mon Sep 17 00:00:00 2001 From: jdaev Date: Fri, 29 Jul 2022 11:13:35 -0400 Subject: [PATCH 12/53] refactor: add comments PE-1904 --- src/parameter_declarations.ts | 15 ++++++++++----- src/utils/temp.ts | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/parameter_declarations.ts b/src/parameter_declarations.ts index 6b043a11..e43661dc 100644 --- a/src/parameter_declarations.ts +++ b/src/parameter_declarations.ts @@ -261,7 +261,8 @@ Parameter.declare({ \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-file-path \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-files \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-paths -\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-csv`, +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-csv +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --remote-path`, forbiddenConjunctionParameters: [LocalFilePathParameter_DEPRECATED, LocalPathsParameter, LocalCSVParameter] }); @@ -294,7 +295,8 @@ Parameter.declare({ \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-path \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-paths \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-csv -\t\t\t\t\t\t\t• Can NOT be used in conjunction with --dest-file-name`, +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --dest-file-name +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --remote-path`, forbiddenConjunctionParameters: [ LocalFilePathParameter_DEPRECATED, LocalPathParameter, @@ -395,7 +397,8 @@ Parameter.declare({ \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-file-path \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-files \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-paths -\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-csv`, +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-csv +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --remote-path`, forbiddenConjunctionParameters: [ LocalFilePathParameter_DEPRECATED, LocalFilesParameter_DEPRECATED, @@ -413,7 +416,8 @@ Parameter.declare({ \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-files \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-path \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-csv -\t\t\t\t\t\t\t• Can NOT be used in conjunction with --dest-file-name`, +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --dest-file-name +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --remote-path`, forbiddenConjunctionParameters: [ LocalFilePathParameter_DEPRECATED, LocalFilesParameter_DEPRECATED, @@ -439,7 +443,8 @@ Parameter.declare({ \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-files \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-path \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-paths -\t\t\t\t\t\t\t• Can NOT be used in conjunction with --dest-file-name`, +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --dest-file-name +\t\t\t\t\t\t\t• Can NOT be used in conjunction with --remote-path`, forbiddenConjunctionParameters: [ LocalFilePathParameter_DEPRECATED, LocalFilesParameter_DEPRECATED, diff --git a/src/utils/temp.ts b/src/utils/temp.ts index 6ccec845..dcbc5302 100644 --- a/src/utils/temp.ts +++ b/src/utils/temp.ts @@ -17,7 +17,7 @@ export function getTempFolder(): string { return tempFolderPath; } - fs.mkdirSync(tempFolderPath); + fs.mkdirSync(tempFolderPath, { recursive: true }); return tempFolderPath; } From 99fa24e9c0f8489d5ce5ac723a444fb196b7f516 Mon Sep 17 00:00:00 2001 From: jdaev Date: Fri, 29 Jul 2022 11:58:20 -0400 Subject: [PATCH 13/53] test: add download test PE-1904 --- src/commands/upload_file.ts | 4 ---- src/utils/download.test.ts | 13 +++++++++++++ src/utils/download.ts | 3 +-- 3 files changed, 14 insertions(+), 6 deletions(-) create mode 100644 src/utils/download.test.ts diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index 05fe6430..52318221 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -134,10 +134,6 @@ async function getRemoteFile( const localFilePath = await download(remoteFilePath, tempFolder); - console.log(localFilePath); - if (!localFilePath) { - return undefined; - } const customContentType = parameters.getParameterValue(CustomContentTypeParameter); const wrappedEntity = wrapFileOrFolder(localFilePath, customContentType); diff --git a/src/utils/download.test.ts b/src/utils/download.test.ts new file mode 100644 index 00000000..8372c5e8 --- /dev/null +++ b/src/utils/download.test.ts @@ -0,0 +1,13 @@ +import { expect } from 'chai'; +import { getTempFolder } from './temp'; +import { download } from './download'; +import * as fs from 'fs'; + +describe('download test', () => { + const validDownloadLink = 'https://arweave.net/pVoSqZgJUCiNw7oS6CtlVEd8gREQlpRbccrsMLkeIuQ'; + const tempFolderPath = getTempFolder(); + it('downloads a file into the provided folder when given a valid link', async () => { + const result = await download(validDownloadLink, tempFolderPath); + expect(fs.existsSync(result)).to.equal(true); + }); +}); diff --git a/src/utils/download.ts b/src/utils/download.ts index 1bcdf40d..660dfed1 100644 --- a/src/utils/download.ts +++ b/src/utils/download.ts @@ -8,14 +8,13 @@ import axios, { AxiosResponse } from 'axios'; * @param url URL of the file to download. * @param destinationPath Path to the destination file. */ -export async function download(url: string, filePath: string): Promise { +export async function download(url: string, filePath: string): Promise { const fileName = url.split('/').pop(); if (!fileName) { throw new Error('Invalid remote path. No file name found.'); } try { const response: AxiosResponse = await axios.get(url, { responseType: 'arraybuffer' }); - console.log(response.headers['content-type']); const pathToFile = path.join(filePath, fileName); fs.writeFileSync(pathToFile, response.data); From cbd3819e248856ad83568a390dc3fc5e90eae0b4 Mon Sep 17 00:00:00 2001 From: jdaev Date: Fri, 29 Jul 2022 12:02:00 -0400 Subject: [PATCH 14/53] test: add invalid download test PE-1904 --- src/utils/download.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/utils/download.test.ts b/src/utils/download.test.ts index 8372c5e8..e6875cba 100644 --- a/src/utils/download.test.ts +++ b/src/utils/download.test.ts @@ -5,9 +5,20 @@ import * as fs from 'fs'; describe('download test', () => { const validDownloadLink = 'https://arweave.net/pVoSqZgJUCiNw7oS6CtlVEd8gREQlpRbccrsMLkeIuQ'; + const invalidDownloadLink = 'https://arweave.net/pVoSqZgJUCiNw7oS6CtlVEV8gREQlpRbccrsMLkeIuQ'; const tempFolderPath = getTempFolder(); it('downloads a file into the provided folder when given a valid link', async () => { const result = await download(validDownloadLink, tempFolderPath); expect(fs.existsSync(result)).to.equal(true); }); + + it('download throws when given an invalid link', async () => { + let error; + try { + await download(invalidDownloadLink, tempFolderPath); + } catch (err) { + error = err; + } + expect(error?.name).to.equal('Error'); + }); }); From 608e109cb5ed8d387fb447430663d03ea628934a Mon Sep 17 00:00:00 2001 From: jdaev Date: Fri, 29 Jul 2022 12:10:33 -0400 Subject: [PATCH 15/53] refactor: add temp folder cleanup function PE-1904 --- src/commands/upload_file.ts | 3 ++- src/utils/temp.ts | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index caf5fbff..1910d135 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -33,7 +33,7 @@ import { import { cliArDriveFactory } from '..'; import * as fs from 'fs'; import { getArweaveFromURL } from '../utils/get_arweave_for_url'; -import { getTempFolder } from '../utils/temp'; +import { cleanUpTempFolder, getTempFolder } from '../utils/temp'; import { download } from '../utils/download'; interface UploadPathParameter { @@ -248,6 +248,7 @@ new CLICommand({ }); console.log(JSON.stringify(results, null, 4)); + cleanUpTempFolder(); return SUCCESS_EXIT_CODE; } diff --git a/src/utils/temp.ts b/src/utils/temp.ts index dcbc5302..67e6be05 100644 --- a/src/utils/temp.ts +++ b/src/utils/temp.ts @@ -21,3 +21,15 @@ export function getTempFolder(): string { return tempFolderPath; } + +export function cleanUpTempFolder(): void { + const tempFolderPath = platformTempFolder(); + fs.readdir(tempFolderPath, (err, files) => { + if (err) throw err; + for (const file of files) { + fs.unlink(path.join(tempFolderPath, file), (err) => { + if (err) throw err; + }); + } + }); +} From d003b917e848822b4f7a97727955d1b078ba59e1 Mon Sep 17 00:00:00 2001 From: Javed Hussein Date: Mon, 1 Aug 2022 10:41:07 -0400 Subject: [PATCH 16/53] Update src/utils/temp.ts Co-authored-by: Derek Sonnenberg --- src/utils/temp.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/utils/temp.ts b/src/utils/temp.ts index 67e6be05..0485dc31 100644 --- a/src/utils/temp.ts +++ b/src/utils/temp.ts @@ -25,10 +25,14 @@ export function getTempFolder(): string { export function cleanUpTempFolder(): void { const tempFolderPath = platformTempFolder(); fs.readdir(tempFolderPath, (err, files) => { - if (err) throw err; + if (err) { + throw err; + } for (const file of files) { fs.unlink(path.join(tempFolderPath, file), (err) => { - if (err) throw err; + if (err) { + throw err; + } }); } }); From 7797479c469147848ea9828bfd1402a6cc3fc7ea Mon Sep 17 00:00:00 2001 From: Javed Hussein Date: Mon, 1 Aug 2022 10:42:00 -0400 Subject: [PATCH 17/53] Update src/utils/download.test.ts Co-authored-by: Derek Sonnenberg --- src/utils/download.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/utils/download.test.ts b/src/utils/download.test.ts index e6875cba..44d2e0ce 100644 --- a/src/utils/download.test.ts +++ b/src/utils/download.test.ts @@ -20,5 +20,6 @@ describe('download test', () => { error = err; } expect(error?.name).to.equal('Error'); + expect(error?.message).to.equal('Invalid remote path. No file name found.'); }); }); From 0caf271e882d332dd29d15e968b4265383478b6f Mon Sep 17 00:00:00 2001 From: Javed Hussein Date: Mon, 1 Aug 2022 10:52:09 -0400 Subject: [PATCH 18/53] refactor: resolve nits PE-1904 --- src/commands/upload_file.ts | 1 - src/utils/download.ts | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index 1910d135..ce3c0af8 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -189,7 +189,6 @@ new CLICommand({ return fileList; } const filesFromRemote = await getRemoteFile(parameters, parentFolderId); - console.log(filesFromRemote); if (filesFromRemote) { return filesFromRemote; } diff --git a/src/utils/download.ts b/src/utils/download.ts index 660dfed1..50203ff3 100644 --- a/src/utils/download.ts +++ b/src/utils/download.ts @@ -20,6 +20,8 @@ export async function download(url: string, filePath: string): Promise { fs.writeFileSync(pathToFile, response.data); return pathToFile; } catch (error) { - throw new Error(`Failed to download file from remote path ${url}: ${error.message}`); + throw new Error( + 'Failed to download file from remote path https://arweave.net/pVoSqZgJUCiNw7oS6CtlVEV8gREQlpRbccrsMLkeIuQ: Request failed with status code 404' + ); } } From 1f5d81c4cc7dd183e867046474b0e2ac2264ca8a Mon Sep 17 00:00:00 2001 From: Javed Hussein Date: Mon, 1 Aug 2022 10:55:46 -0400 Subject: [PATCH 19/53] refactor: update error and test PE-1904 --- src/utils/download.test.ts | 4 +++- src/utils/download.ts | 4 +--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/utils/download.test.ts b/src/utils/download.test.ts index 44d2e0ce..6c889790 100644 --- a/src/utils/download.test.ts +++ b/src/utils/download.test.ts @@ -20,6 +20,8 @@ describe('download test', () => { error = err; } expect(error?.name).to.equal('Error'); - expect(error?.message).to.equal('Invalid remote path. No file name found.'); + expect(error?.message).to.equal( + 'Failed to download file from remote path https://arweave.net/pVoSqZgJUCiNw7oS6CtlVEV8gREQlpRbccrsMLkeIuQ: Request failed with status code 404' + ); }); }); diff --git a/src/utils/download.ts b/src/utils/download.ts index 50203ff3..660dfed1 100644 --- a/src/utils/download.ts +++ b/src/utils/download.ts @@ -20,8 +20,6 @@ export async function download(url: string, filePath: string): Promise { fs.writeFileSync(pathToFile, response.data); return pathToFile; } catch (error) { - throw new Error( - 'Failed to download file from remote path https://arweave.net/pVoSqZgJUCiNw7oS6CtlVEV8gREQlpRbccrsMLkeIuQ: Request failed with status code 404' - ); + throw new Error(`Failed to download file from remote path ${url}: ${error.message}`); } } From 2d0e2af2fa9ea39556251906a7618b63e3e6a7de Mon Sep 17 00:00:00 2001 From: jdaev Date: Mon, 1 Aug 2022 16:44:11 -0400 Subject: [PATCH 20/53] refactor: download file as stream PE-1904 --- src/utils/download.ts | 42 ++++++++++++++++++++++++++++++++---------- src/utils/temp.ts | 8 ++++---- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/src/utils/download.ts b/src/utils/download.ts index 660dfed1..5baa8e9d 100644 --- a/src/utils/download.ts +++ b/src/utils/download.ts @@ -1,24 +1,46 @@ import * as fs from 'fs'; import path from 'path'; -import axios, { AxiosResponse } from 'axios'; +import axios from 'axios'; +import { uniqueId } from 'lodash'; /** * Downloads file from remote HTTP[S] host and puts its contents to the * specified location. * @param url URL of the file to download. * @param destinationPath Path to the destination file. + * @param fileName The file name. If no name is provided a random name will be used. */ -export async function download(url: string, filePath: string): Promise { - const fileName = url.split('/').pop(); - if (!fileName) { - throw new Error('Invalid remote path. No file name found.'); - } +export async function download(url: string, destinationPath: string, fileName?: string): Promise { + const name = fileName ?? url.split('/').pop() ?? uniqueId(); + const pathToFile = path.join(destinationPath, name); try { - const response: AxiosResponse = await axios.get(url, { responseType: 'arraybuffer' }); - const pathToFile = path.join(filePath, fileName); + const writer = fs.createWriteStream(pathToFile); + + return axios({ + method: 'get', + url: url, + responseType: 'stream' + }).then((response) => { + //ensure that the user can call `then()` only when the file has + //been downloaded entirely. - fs.writeFileSync(pathToFile, response.data); - return pathToFile; + return new Promise((resolve, reject) => { + response.data.pipe(writer); + let error: Error | null = null; + writer.on('error', (err) => { + error = err; + writer.close(); + reject(err); + }); + writer.on('close', () => { + if (!error) { + resolve(pathToFile); + } + //no need to call the reject here, as it will have been called in the + //'error' stream; + }); + }); + }); } catch (error) { throw new Error(`Failed to download file from remote path ${url}: ${error.message}`); } diff --git a/src/utils/temp.ts b/src/utils/temp.ts index 0485dc31..2b5c0b00 100644 --- a/src/utils/temp.ts +++ b/src/utils/temp.ts @@ -25,14 +25,14 @@ export function getTempFolder(): string { export function cleanUpTempFolder(): void { const tempFolderPath = platformTempFolder(); fs.readdir(tempFolderPath, (err, files) => { - if (err) { - throw err; + if (err) { + throw err; } for (const file of files) { fs.unlink(path.join(tempFolderPath, file), (err) => { if (err) { - throw err; - } + throw err; + } }); } }); From 9b2aa5dd50735f54c8683cb7d17e5abdf7b45dcb Mon Sep 17 00:00:00 2001 From: Javed Hussein Date: Mon, 1 Aug 2022 17:18:24 -0400 Subject: [PATCH 21/53] refactor: add download progress PE-1904 --- src/commands/upload_file.ts | 4 +++- src/utils/download.ts | 12 ++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index ce3c0af8..7971b420 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -135,7 +135,9 @@ async function getRemoteFile( const tempFolder = getTempFolder(); - const localFilePath = await download(remoteFilePath, tempFolder); + const localFilePath = await download(remoteFilePath, tempFolder, (downloadProgress: number) => { + console.log('Downloading file...', downloadProgress); + }); const customContentType = parameters.getParameterValue(CustomContentTypeParameter); diff --git a/src/utils/download.ts b/src/utils/download.ts index 5baa8e9d..169818ab 100644 --- a/src/utils/download.ts +++ b/src/utils/download.ts @@ -3,6 +3,8 @@ import path from 'path'; import axios from 'axios'; import { uniqueId } from 'lodash'; +type DownloadProgressCallback = (downloadProgress: number) => void; + /** * Downloads file from remote HTTP[S] host and puts its contents to the * specified location. @@ -10,7 +12,12 @@ import { uniqueId } from 'lodash'; * @param destinationPath Path to the destination file. * @param fileName The file name. If no name is provided a random name will be used. */ -export async function download(url: string, destinationPath: string, fileName?: string): Promise { +export async function download( + url: string, + destinationPath: string, + downloadProgressCallback?: DownloadProgressCallback, + fileName?: string +): Promise { const name = fileName ?? url.split('/').pop() ?? uniqueId(); const pathToFile = path.join(destinationPath, name); try { @@ -19,7 +26,8 @@ export async function download(url: string, destinationPath: string, fileName?: return axios({ method: 'get', url: url, - responseType: 'stream' + responseType: 'stream', + onDownloadProgress: downloadProgressCallback }).then((response) => { //ensure that the user can call `then()` only when the file has //been downloaded entirely. From dfba33fde37ae060a55d6b0db7c36561b15df46e Mon Sep 17 00:00:00 2001 From: Javed Hussein Date: Mon, 1 Aug 2022 17:48:56 -0400 Subject: [PATCH 22/53] refactor: use uuid for random name PE-1904 --- src/commands/upload_file.ts | 2 +- src/utils/download.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index 7971b420..6d081b56 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -136,7 +136,7 @@ async function getRemoteFile( const tempFolder = getTempFolder(); const localFilePath = await download(remoteFilePath, tempFolder, (downloadProgress: number) => { - console.log('Downloading file...', downloadProgress); + console.error('Downloading file...', downloadProgress); }); const customContentType = parameters.getParameterValue(CustomContentTypeParameter); diff --git a/src/utils/download.ts b/src/utils/download.ts index 169818ab..3c9e2482 100644 --- a/src/utils/download.ts +++ b/src/utils/download.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import path from 'path'; import axios from 'axios'; -import { uniqueId } from 'lodash'; +import { randomUUID } from 'crypto'; type DownloadProgressCallback = (downloadProgress: number) => void; @@ -18,7 +18,7 @@ export async function download( downloadProgressCallback?: DownloadProgressCallback, fileName?: string ): Promise { - const name = fileName ?? url.split('/').pop() ?? uniqueId(); + const name = fileName ?? url.split('/').pop() ?? randomUUID(); const pathToFile = path.join(destinationPath, name); try { const writer = fs.createWriteStream(pathToFile); From a21dd90c0bfa5e1992208acb4024fff1b79d9082 Mon Sep 17 00:00:00 2001 From: jdaev Date: Tue, 2 Aug 2022 10:01:40 -0400 Subject: [PATCH 23/53] refactor: add download progress log PW-1904 --- src/commands/upload_file.ts | 2 +- src/utils/download.ts | 45 ++++++++++++++++++------------------- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index 6d081b56..de98b687 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -136,7 +136,7 @@ async function getRemoteFile( const tempFolder = getTempFolder(); const localFilePath = await download(remoteFilePath, tempFolder, (downloadProgress: number) => { - console.error('Downloading file...', downloadProgress); + process.stderr.write('Downloading file... ' + downloadProgress + '\r'); }); const customContentType = parameters.getParameterValue(CustomContentTypeParameter); diff --git a/src/utils/download.ts b/src/utils/download.ts index 3c9e2482..6940d776 100644 --- a/src/utils/download.ts +++ b/src/utils/download.ts @@ -20,36 +20,35 @@ export async function download( ): Promise { const name = fileName ?? url.split('/').pop() ?? randomUUID(); const pathToFile = path.join(destinationPath, name); + const writer = fs.createWriteStream(pathToFile); try { - const writer = fs.createWriteStream(pathToFile); - - return axios({ + const { data, headers } = await axios({ method: 'get', url: url, - responseType: 'stream', - onDownloadProgress: downloadProgressCallback - }).then((response) => { - //ensure that the user can call `then()` only when the file has - //been downloaded entirely. + responseType: 'stream' + }); + const totalLength = headers['content-length']; - return new Promise((resolve, reject) => { - response.data.pipe(writer); - let error: Error | null = null; - writer.on('error', (err) => { - error = err; - writer.close(); - reject(err); - }); - writer.on('close', () => { - if (!error) { - resolve(pathToFile); - } - //no need to call the reject here, as it will have been called in the - //'error' stream; - }); + data.on( + 'data', + (chunk: string | unknown[]) => + downloadProgressCallback && downloadProgressCallback((chunk.length / totalLength) * 100) + ); + return new Promise((resolve) => { + data.pipe(writer); + let error: Error | null = null; + writer.on('error', (err) => { + error = err; + writer.close(); + }); + writer.on('close', () => { + if (!error) { + resolve(pathToFile); + } }); }); } catch (error) { + writer.close(); throw new Error(`Failed to download file from remote path ${url}: ${error.message}`); } } From b827814645ff0d99d4901bd7ebcee3e8b213771a Mon Sep 17 00:00:00 2001 From: Javed Hussein Date: Tue, 2 Aug 2022 11:41:29 -0400 Subject: [PATCH 24/53] refactor: make fileName required test: add cleanupTemp test PE-1904 --- src/commands/upload_file.ts | 14 ++++++++++---- src/parameter_declarations.ts | 6 ++++-- src/utils/download.test.ts | 5 +++-- src/utils/download.ts | 10 ++++------ src/utils/temp.test.ts | 9 ++++++++- 5 files changed, 29 insertions(+), 15 deletions(-) diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index de98b687..989b6ef7 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -134,10 +134,16 @@ async function getRemoteFile( } const tempFolder = getTempFolder(); - - const localFilePath = await download(remoteFilePath, tempFolder, (downloadProgress: number) => { - process.stderr.write('Downloading file... ' + downloadProgress + '\r'); - }); + const destinationFileName = parameters.getRequiredParameterValue(DestinationFileNameParameter); + + const localFilePath = await download( + remoteFilePath, + tempFolder, + destinationFileName, + (downloadProgress: number) => { + process.stderr.write('Downloading file...' + downloadProgress + '%\r'); + } + ); const customContentType = parameters.getParameterValue(CustomContentTypeParameter); diff --git a/src/parameter_declarations.ts b/src/parameter_declarations.ts index 5cadfaed..70f0e804 100644 --- a/src/parameter_declarations.ts +++ b/src/parameter_declarations.ts @@ -284,8 +284,9 @@ Parameter.declare({ Parameter.declare({ name: DestinationFileNameParameter, aliases: ['-d', '--dest-file-name'], - description: `(OPTIONAL) a destination file name to use when uploaded to ArDrive -\t\t\t\t\t\t\t• Only valid for use with --local-path or --local-file-path` + description: `a destination file name to use when uploaded to ArDrive +\t\t\t\t\t\t\t• Required for use with --remote-path +\t\t\t\t\t\t\t• Optional use with --local-path or --local-file-path` }); Parameter.declare({ @@ -493,6 +494,7 @@ Parameter.declare({ name: RemotePathParameter, aliases: ['--remote-path'], description: `the remote path for the file that will be uploaded +\t\t\t\t\t\t\t• MUST be used in conjunction with --dest-file-name \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-file-path \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-files \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-paths diff --git a/src/utils/download.test.ts b/src/utils/download.test.ts index 6c889790..995fcbb8 100644 --- a/src/utils/download.test.ts +++ b/src/utils/download.test.ts @@ -6,16 +6,17 @@ import * as fs from 'fs'; describe('download test', () => { const validDownloadLink = 'https://arweave.net/pVoSqZgJUCiNw7oS6CtlVEd8gREQlpRbccrsMLkeIuQ'; const invalidDownloadLink = 'https://arweave.net/pVoSqZgJUCiNw7oS6CtlVEV8gREQlpRbccrsMLkeIuQ'; + const name = 'cat.jpg'; const tempFolderPath = getTempFolder(); it('downloads a file into the provided folder when given a valid link', async () => { - const result = await download(validDownloadLink, tempFolderPath); + const result = await download(validDownloadLink, tempFolderPath, name); expect(fs.existsSync(result)).to.equal(true); }); it('download throws when given an invalid link', async () => { let error; try { - await download(invalidDownloadLink, tempFolderPath); + await download(invalidDownloadLink, tempFolderPath, name); } catch (err) { error = err; } diff --git a/src/utils/download.ts b/src/utils/download.ts index 6940d776..4d93ae91 100644 --- a/src/utils/download.ts +++ b/src/utils/download.ts @@ -1,7 +1,6 @@ import * as fs from 'fs'; import path from 'path'; import axios from 'axios'; -import { randomUUID } from 'crypto'; type DownloadProgressCallback = (downloadProgress: number) => void; @@ -10,16 +9,15 @@ type DownloadProgressCallback = (downloadProgress: number) => void; * specified location. * @param url URL of the file to download. * @param destinationPath Path to the destination file. - * @param fileName The file name. If no name is provided a random name will be used. + * @param destinationFileName The file name. */ export async function download( url: string, destinationPath: string, - downloadProgressCallback?: DownloadProgressCallback, - fileName?: string + destinationFileName: string, + downloadProgressCallback?: DownloadProgressCallback ): Promise { - const name = fileName ?? url.split('/').pop() ?? randomUUID(); - const pathToFile = path.join(destinationPath, name); + const pathToFile = path.join(destinationPath, destinationFileName); const writer = fs.createWriteStream(pathToFile); try { const { data, headers } = await axios({ diff --git a/src/utils/temp.test.ts b/src/utils/temp.test.ts index b0f4dab6..9e24ecdf 100644 --- a/src/utils/temp.test.ts +++ b/src/utils/temp.test.ts @@ -1,5 +1,5 @@ import { expect } from 'chai'; -import { getTempFolder } from './temp'; +import { cleanUpTempFolder, getTempFolder } from './temp'; import * as fs from 'fs'; describe('temp folder test', () => { @@ -13,4 +13,11 @@ describe('temp folder test', () => { expect(tempFolderPath).to.contains('ardrive'); expect(tempFolderPath).to.contains('temp'); }); + + it('cleanUpTempFolder removes the temporary folder from the local system', () => { + const tempFolderPath = getTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(true); + cleanUpTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(false); + }); }); From 134f67c0c7cc0b4d98d915dc954fbfc4d79c0912 Mon Sep 17 00:00:00 2001 From: jdaev Date: Tue, 2 Aug 2022 12:28:05 -0400 Subject: [PATCH 25/53] fix: fix download progress PE-1904 --- src/commands/upload_file.ts | 2 +- src/utils/download.ts | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index 989b6ef7..949d93e6 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -141,7 +141,7 @@ async function getRemoteFile( tempFolder, destinationFileName, (downloadProgress: number) => { - process.stderr.write('Downloading file...' + downloadProgress + '%\r'); + process.stderr.write('Downloading file...' + downloadProgress.toFixed(2) + '\r'); } ); diff --git a/src/utils/download.ts b/src/utils/download.ts index 4d93ae91..8841abc2 100644 --- a/src/utils/download.ts +++ b/src/utils/download.ts @@ -26,12 +26,11 @@ export async function download( responseType: 'stream' }); const totalLength = headers['content-length']; - - data.on( - 'data', - (chunk: string | unknown[]) => - downloadProgressCallback && downloadProgressCallback((chunk.length / totalLength) * 100) - ); + let downloadedLength = 0; + data.on('data', (chunk: string | unknown[]) => { + downloadedLength += chunk.length; + downloadProgressCallback && downloadProgressCallback((downloadedLength / totalLength) * 100); + }); return new Promise((resolve) => { data.pipe(writer); let error: Error | null = null; From 47a52cc25ad6eac33925d82301b43b380fa587b0 Mon Sep 17 00:00:00 2001 From: jdaev Date: Tue, 2 Aug 2022 12:47:09 -0400 Subject: [PATCH 26/53] test: fix cleanup test PE-1904 --- src/commands/upload_file.ts | 4 ++-- src/utils/download.test.ts | 5 +++-- src/utils/download.ts | 6 ++++-- src/utils/temp.ts | 13 +------------ 4 files changed, 10 insertions(+), 18 deletions(-) diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index 949d93e6..12309a45 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -136,7 +136,7 @@ async function getRemoteFile( const tempFolder = getTempFolder(); const destinationFileName = parameters.getRequiredParameterValue(DestinationFileNameParameter); - const localFilePath = await download( + const { pathToFile, contentType } = await download( remoteFilePath, tempFolder, destinationFileName, @@ -147,7 +147,7 @@ async function getRemoteFile( const customContentType = parameters.getParameterValue(CustomContentTypeParameter); - const wrappedEntity = wrapFileOrFolder(localFilePath, customContentType); + const wrappedEntity = wrapFileOrFolder(pathToFile, customContentType ?? contentType); const singleParameter = { parentFolderId: parentFolderId, wrappedEntity, diff --git a/src/utils/download.test.ts b/src/utils/download.test.ts index 995fcbb8..8a659080 100644 --- a/src/utils/download.test.ts +++ b/src/utils/download.test.ts @@ -9,8 +9,9 @@ describe('download test', () => { const name = 'cat.jpg'; const tempFolderPath = getTempFolder(); it('downloads a file into the provided folder when given a valid link', async () => { - const result = await download(validDownloadLink, tempFolderPath, name); - expect(fs.existsSync(result)).to.equal(true); + const { pathToFile, contentType } = await download(validDownloadLink, tempFolderPath, name); + expect(fs.existsSync(pathToFile)).to.equal(true); + expect(contentType).to.equal('image/jpeg'); }); it('download throws when given an invalid link', async () => { diff --git a/src/utils/download.ts b/src/utils/download.ts index 8841abc2..5f8da06d 100644 --- a/src/utils/download.ts +++ b/src/utils/download.ts @@ -3,6 +3,7 @@ import path from 'path'; import axios from 'axios'; type DownloadProgressCallback = (downloadProgress: number) => void; +type DownloadResult = { pathToFile: string; contentType: string }; /** * Downloads file from remote HTTP[S] host and puts its contents to the @@ -16,7 +17,7 @@ export async function download( destinationPath: string, destinationFileName: string, downloadProgressCallback?: DownloadProgressCallback -): Promise { +): Promise { const pathToFile = path.join(destinationPath, destinationFileName); const writer = fs.createWriteStream(pathToFile); try { @@ -26,6 +27,7 @@ export async function download( responseType: 'stream' }); const totalLength = headers['content-length']; + const contentType = headers['content-type']; let downloadedLength = 0; data.on('data', (chunk: string | unknown[]) => { downloadedLength += chunk.length; @@ -40,7 +42,7 @@ export async function download( }); writer.on('close', () => { if (!error) { - resolve(pathToFile); + resolve({ pathToFile, contentType }); } }); }); diff --git a/src/utils/temp.ts b/src/utils/temp.ts index 2b5c0b00..948d0d8f 100644 --- a/src/utils/temp.ts +++ b/src/utils/temp.ts @@ -24,16 +24,5 @@ export function getTempFolder(): string { export function cleanUpTempFolder(): void { const tempFolderPath = platformTempFolder(); - fs.readdir(tempFolderPath, (err, files) => { - if (err) { - throw err; - } - for (const file of files) { - fs.unlink(path.join(tempFolderPath, file), (err) => { - if (err) { - throw err; - } - }); - } - }); + fs.rmdirSync(tempFolderPath, { recursive: true }); } From 88fe85a5c6debb19a88aa22f194c5317bfd1ea24 Mon Sep 17 00:00:00 2001 From: jdaev Date: Tue, 2 Aug 2022 12:52:18 -0400 Subject: [PATCH 27/53] refactor: show progress only if enabled PE-1904 --- src/commands/upload_file.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index 12309a45..81f80b7a 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -36,6 +36,8 @@ import { getArweaveFromURL } from '../utils/get_arweave_for_url'; import { cleanUpTempFolder, getTempFolder } from '../utils/temp'; import { download } from '../utils/download'; +const ARDRIVE_PROGRESS_LOG = 'ARDRIVE_PROGRESS_LOG'; + interface UploadPathParameter { parentFolderId: FolderID; wrappedEntity: ArFSFileToUpload | ArFSFolderToUpload; @@ -141,7 +143,9 @@ async function getRemoteFile( tempFolder, destinationFileName, (downloadProgress: number) => { - process.stderr.write('Downloading file...' + downloadProgress.toFixed(2) + '\r'); + if (process.env[ARDRIVE_PROGRESS_LOG] && process.env[ARDRIVE_PROGRESS_LOG] === '1') { + process.stderr.write('Downloading file...' + downloadProgress.toFixed(2) + '\r'); + } } ); From de7357b92cb3884a9762c53eec3aac7ecd9a5d5e Mon Sep 17 00:00:00 2001 From: jdaev Date: Tue, 2 Aug 2022 13:08:01 -0400 Subject: [PATCH 28/53] chore: update readme PE-1904 --- README.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9c9ef331..32ab53a7 100644 --- a/README.md +++ b/README.md @@ -129,9 +129,10 @@ ardrive upload-file --wallet-file /path/to/my/wallet.json --parent-folder-id "f0 15. [Uploading Manifests](#uploading-manifests) 16. [Hosting a Webpage with Manifest](#hosting-a-webpage-with-manifest) 17. [Uploading With a Custom Content Type](#custom-content-type) - 18. [Uploading a Custom Manifest](#custom-manifest) - 19. [Uploading Files with Custom MetaData](#uploading-files-with-custom-metadata) - 20. [Applying Unique Custom MetaData During Bulk Workflows](#applying-unique-custom-metadata-during-bulk-workflows) + 18. [Uploading From a Remote URL](#remote-path) + 19. [Uploading a Custom Manifest](#custom-manifest) + 20. [Uploading Files with Custom MetaData](#uploading-files-with-custom-metadata) + 21. [Applying Unique Custom MetaData During Bulk Workflows](#applying-unique-custom-metadata-during-bulk-workflows) 8. [Other Utility Operations](#other-utility-operations) 1. [Monitoring Transactions](#monitoring-transactions) 2. [Dealing With Network Congestion](#dealing-with-network-congestion) @@ -1136,6 +1137,16 @@ It is currently possible to set this value to any given string, but the gateway Note: In the case of multi-file uploads or recursive folder uploads, setting this `--content-type` flag will set the provided custom content type on EVERY file entity within a given upload. +### Uploading From a Remote URL + +You can upload a file from an existing url using the `--remote-path` flag. This must be used in conjunction with `--dest-file-name` + +You can use a custom content type using the `--content-type` flag, but if this isn't used the app will use the content type from the response header: + +```shell +ardrive upload-file --remote-path "https://url/to/file" --parent-folder-id "9af694f6-4cfc-4eee-88a8-1b02704760c0" -d "example.jpg" -w /path/to/wallet.json +``` + ### Uploading a Custom Manifest Using the custom content type feature, it is possible for users to upload their own custom manifests. The Arweave gateways use this special content type in order to identify an uploaded file as a manifest: From 1d8ef8de2798f9406fededc362993fdf74482205 Mon Sep 17 00:00:00 2001 From: Javed Hussein Date: Tue, 2 Aug 2022 15:16:36 -0400 Subject: [PATCH 29/53] refactor: move env var check to utils PE-1904 --- src/commands/upload_file.ts | 7 +++---- src/utils/download.ts | 1 + src/utils/show_progress_log.ts | 3 +++ 3 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 src/utils/show_progress_log.ts diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index 81f80b7a..c0c01cb0 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -35,8 +35,7 @@ import * as fs from 'fs'; import { getArweaveFromURL } from '../utils/get_arweave_for_url'; import { cleanUpTempFolder, getTempFolder } from '../utils/temp'; import { download } from '../utils/download'; - -const ARDRIVE_PROGRESS_LOG = 'ARDRIVE_PROGRESS_LOG'; +import { showProgressLog } from '../utils/show_progress_log'; interface UploadPathParameter { parentFolderId: FolderID; @@ -143,8 +142,8 @@ async function getRemoteFile( tempFolder, destinationFileName, (downloadProgress: number) => { - if (process.env[ARDRIVE_PROGRESS_LOG] && process.env[ARDRIVE_PROGRESS_LOG] === '1') { - process.stderr.write('Downloading file...' + downloadProgress.toFixed(2) + '\r'); + if (showProgressLog) { + process.stderr.write(`Downloading file... ${downloadProgress}\r`); } } ); diff --git a/src/utils/download.ts b/src/utils/download.ts index 5f8da06d..1b108421 100644 --- a/src/utils/download.ts +++ b/src/utils/download.ts @@ -12,6 +12,7 @@ type DownloadResult = { pathToFile: string; contentType: string }; * @param destinationPath Path to the destination file. * @param destinationFileName The file name. */ + export async function download( url: string, destinationPath: string, diff --git a/src/utils/show_progress_log.ts b/src/utils/show_progress_log.ts new file mode 100644 index 00000000..16393c25 --- /dev/null +++ b/src/utils/show_progress_log.ts @@ -0,0 +1,3 @@ +const ARDRIVE_PROGRESS_LOG = 'ARDRIVE_PROGRESS_LOG'; + +export const showProgressLog = process.env[ARDRIVE_PROGRESS_LOG] && process.env[ARDRIVE_PROGRESS_LOG] === '1'; From 013360ad37b7558b5e876ae3785a2489f76630e2 Mon Sep 17 00:00:00 2001 From: jdaev Date: Tue, 2 Aug 2022 17:09:45 -0400 Subject: [PATCH 30/53] refactor: refactor file names PE-1904 --- src/commands/upload_file.ts | 10 +++++----- src/utils/{download.test.ts => download_file.test.ts} | 10 +++++----- src/utils/{download.ts => download_file.ts} | 2 +- src/utils/{temp.test.ts => temp_folder.test.ts} | 2 +- src/utils/{temp.ts => temp_folder.ts} | 0 5 files changed, 12 insertions(+), 12 deletions(-) rename src/utils/{download.test.ts => download_file.test.ts} (74%) rename src/utils/{download.ts => download_file.ts} (97%) rename src/utils/{temp.test.ts => temp_folder.test.ts} (91%) rename src/utils/{temp.ts => temp_folder.ts} (100%) diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index c0c01cb0..d2873211 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -33,8 +33,8 @@ import { import { cliArDriveFactory } from '..'; import * as fs from 'fs'; import { getArweaveFromURL } from '../utils/get_arweave_for_url'; -import { cleanUpTempFolder, getTempFolder } from '../utils/temp'; -import { download } from '../utils/download'; +import { cleanUpTempFolder, getTempFolder } from '../utils/temp_folder'; +import { downloadFile } from '../utils/download_file'; import { showProgressLog } from '../utils/show_progress_log'; interface UploadPathParameter { @@ -137,13 +137,13 @@ async function getRemoteFile( const tempFolder = getTempFolder(); const destinationFileName = parameters.getRequiredParameterValue(DestinationFileNameParameter); - const { pathToFile, contentType } = await download( + const { pathToFile, contentType } = await downloadFile( remoteFilePath, tempFolder, destinationFileName, (downloadProgress: number) => { - if (showProgressLog) { - process.stderr.write(`Downloading file... ${downloadProgress}\r`); + if (!showProgressLog) { + process.stderr.write(`Downloading file... ${downloadProgress.toFixed(2)}% \r`); } } ); diff --git a/src/utils/download.test.ts b/src/utils/download_file.test.ts similarity index 74% rename from src/utils/download.test.ts rename to src/utils/download_file.test.ts index 8a659080..1e0f0361 100644 --- a/src/utils/download.test.ts +++ b/src/utils/download_file.test.ts @@ -1,15 +1,15 @@ import { expect } from 'chai'; -import { getTempFolder } from './temp'; -import { download } from './download'; +import { getTempFolder } from './temp_folder'; +import { downloadFile } from './download_file'; import * as fs from 'fs'; -describe('download test', () => { +describe('download file test', () => { const validDownloadLink = 'https://arweave.net/pVoSqZgJUCiNw7oS6CtlVEd8gREQlpRbccrsMLkeIuQ'; const invalidDownloadLink = 'https://arweave.net/pVoSqZgJUCiNw7oS6CtlVEV8gREQlpRbccrsMLkeIuQ'; const name = 'cat.jpg'; const tempFolderPath = getTempFolder(); it('downloads a file into the provided folder when given a valid link', async () => { - const { pathToFile, contentType } = await download(validDownloadLink, tempFolderPath, name); + const { pathToFile, contentType } = await downloadFile(validDownloadLink, tempFolderPath, name); expect(fs.existsSync(pathToFile)).to.equal(true); expect(contentType).to.equal('image/jpeg'); }); @@ -17,7 +17,7 @@ describe('download test', () => { it('download throws when given an invalid link', async () => { let error; try { - await download(invalidDownloadLink, tempFolderPath, name); + await downloadFile(invalidDownloadLink, tempFolderPath, name); } catch (err) { error = err; } diff --git a/src/utils/download.ts b/src/utils/download_file.ts similarity index 97% rename from src/utils/download.ts rename to src/utils/download_file.ts index 1b108421..6edd46e4 100644 --- a/src/utils/download.ts +++ b/src/utils/download_file.ts @@ -13,7 +13,7 @@ type DownloadResult = { pathToFile: string; contentType: string }; * @param destinationFileName The file name. */ -export async function download( +export async function downloadFile( url: string, destinationPath: string, destinationFileName: string, diff --git a/src/utils/temp.test.ts b/src/utils/temp_folder.test.ts similarity index 91% rename from src/utils/temp.test.ts rename to src/utils/temp_folder.test.ts index 9e24ecdf..1ee696a1 100644 --- a/src/utils/temp.test.ts +++ b/src/utils/temp_folder.test.ts @@ -1,5 +1,5 @@ import { expect } from 'chai'; -import { cleanUpTempFolder, getTempFolder } from './temp'; +import { cleanUpTempFolder, getTempFolder } from './temp_folder'; import * as fs from 'fs'; describe('temp folder test', () => { diff --git a/src/utils/temp.ts b/src/utils/temp_folder.ts similarity index 100% rename from src/utils/temp.ts rename to src/utils/temp_folder.ts From 0098dcf4623fd4eb4ba430b0e9d49d13198eac49 Mon Sep 17 00:00:00 2001 From: jdaev Date: Tue, 2 Aug 2022 17:12:53 -0400 Subject: [PATCH 31/53] chore: punctuation PE-1904 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 32ab53a7..97d2bccc 100644 --- a/README.md +++ b/README.md @@ -1139,9 +1139,9 @@ Note: In the case of multi-file uploads or recursive folder uploads, setting thi ### Uploading From a Remote URL -You can upload a file from an existing url using the `--remote-path` flag. This must be used in conjunction with `--dest-file-name` +You can upload a file from an existing url using the `--remote-path` flag. This must be used in conjunction with `--dest-file-name`. -You can use a custom content type using the `--content-type` flag, but if this isn't used the app will use the content type from the response header: +You can use a custom content type using the `--content-type` flag, but if this isn't used the app will use the content type from the response header of the request for the remote data. ```shell ardrive upload-file --remote-path "https://url/to/file" --parent-folder-id "9af694f6-4cfc-4eee-88a8-1b02704760c0" -d "example.jpg" -w /path/to/wallet.json From 0491413327e6d83d50e728d854c08453d8e968fa Mon Sep 17 00:00:00 2001 From: jdaev Date: Tue, 2 Aug 2022 17:18:28 -0400 Subject: [PATCH 32/53] fix: fix conditional PE-1904 --- src/commands/upload_file.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index d2873211..ee69a301 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -142,8 +142,8 @@ async function getRemoteFile( tempFolder, destinationFileName, (downloadProgress: number) => { - if (!showProgressLog) { - process.stderr.write(`Downloading file... ${downloadProgress.toFixed(2)}% \r`); + if (showProgressLog) { + process.stderr.write(`Downloading file... ${downloadProgress.toFixed(1)}% \r`); } } ); From 15a93478694ba423c70e7e846a761e60dd48297b Mon Sep 17 00:00:00 2001 From: Javed Hussein Date: Tue, 2 Aug 2022 17:18:58 -0400 Subject: [PATCH 33/53] chore: update src/parameter_declarations.ts Co-authored-by: arielmelendez --- src/parameter_declarations.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parameter_declarations.ts b/src/parameter_declarations.ts index 70f0e804..00fb643a 100644 --- a/src/parameter_declarations.ts +++ b/src/parameter_declarations.ts @@ -286,7 +286,7 @@ Parameter.declare({ aliases: ['-d', '--dest-file-name'], description: `a destination file name to use when uploaded to ArDrive \t\t\t\t\t\t\t• Required for use with --remote-path -\t\t\t\t\t\t\t• Optional use with --local-path or --local-file-path` +\t\t\t\t\t\t\t• Optional when using with --local-path or --local-file-path` }); Parameter.declare({ From 4f3c750c31441b4e91d89d7f26a68f3882fcc296 Mon Sep 17 00:00:00 2001 From: Javed Hussein Date: Tue, 2 Aug 2022 17:45:55 -0400 Subject: [PATCH 34/53] refactor: use pipeline instead of pipe PE-1904 --- src/utils/download_file.ts | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/utils/download_file.ts b/src/utils/download_file.ts index 6edd46e4..dbd238be 100644 --- a/src/utils/download_file.ts +++ b/src/utils/download_file.ts @@ -1,6 +1,10 @@ import * as fs from 'fs'; import path from 'path'; import axios from 'axios'; +import util from 'util'; +import stream from 'stream'; + +const pipeline = util.promisify(stream.pipeline); type DownloadProgressCallback = (downloadProgress: number) => void; type DownloadResult = { pathToFile: string; contentType: string }; @@ -21,6 +25,7 @@ export async function downloadFile( ): Promise { const pathToFile = path.join(destinationPath, destinationFileName); const writer = fs.createWriteStream(pathToFile); + try { const { data, headers } = await axios({ method: 'get', @@ -34,19 +39,8 @@ export async function downloadFile( downloadedLength += chunk.length; downloadProgressCallback && downloadProgressCallback((downloadedLength / totalLength) * 100); }); - return new Promise((resolve) => { - data.pipe(writer); - let error: Error | null = null; - writer.on('error', (err) => { - error = err; - writer.close(); - }); - writer.on('close', () => { - if (!error) { - resolve({ pathToFile, contentType }); - } - }); - }); + await pipeline(data, writer); + return { pathToFile, contentType }; } catch (error) { writer.close(); throw new Error(`Failed to download file from remote path ${url}: ${error.message}`); From 5654fe1d4d87030586bde51e0e867f835f09b172 Mon Sep 17 00:00:00 2001 From: Javed Hussein Date: Tue, 2 Aug 2022 17:46:51 -0400 Subject: [PATCH 35/53] refactor: add var for download progress PE-1904 --- src/utils/download_file.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/utils/download_file.ts b/src/utils/download_file.ts index dbd238be..db7e57fc 100644 --- a/src/utils/download_file.ts +++ b/src/utils/download_file.ts @@ -37,7 +37,9 @@ export async function downloadFile( let downloadedLength = 0; data.on('data', (chunk: string | unknown[]) => { downloadedLength += chunk.length; - downloadProgressCallback && downloadProgressCallback((downloadedLength / totalLength) * 100); + const downloadProgressPct = totalLength > 0 ? (downloadedLength / totalLength) * 100 : 0; + + downloadProgressCallback && downloadProgressCallback(downloadProgressPct); }); await pipeline(data, writer); return { pathToFile, contentType }; From ab35fd4c644f99d48b73a4618f9ee1a5e94e9cab Mon Sep 17 00:00:00 2001 From: jdaev Date: Wed, 3 Aug 2022 09:03:05 -0400 Subject: [PATCH 36/53] test: update tests PE-1904 refactor: clear console --- src/commands/upload_file.ts | 2 +- src/utils/download_file.test.ts | 8 ++++---- src/utils/temp_folder.test.ts | 26 +++++++++++++++++++++----- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index ee69a301..3d7fe1dd 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -147,7 +147,7 @@ async function getRemoteFile( } } ); - + process.stderr.clearLine(0); const customContentType = parameters.getParameterValue(CustomContentTypeParameter); const wrappedEntity = wrapFileOrFolder(pathToFile, customContentType ?? contentType); diff --git a/src/utils/download_file.test.ts b/src/utils/download_file.test.ts index 1e0f0361..55a31c78 100644 --- a/src/utils/download_file.test.ts +++ b/src/utils/download_file.test.ts @@ -3,13 +3,13 @@ import { getTempFolder } from './temp_folder'; import { downloadFile } from './download_file'; import * as fs from 'fs'; -describe('download file test', () => { +describe('downloadFile function', () => { const validDownloadLink = 'https://arweave.net/pVoSqZgJUCiNw7oS6CtlVEd8gREQlpRbccrsMLkeIuQ'; const invalidDownloadLink = 'https://arweave.net/pVoSqZgJUCiNw7oS6CtlVEV8gREQlpRbccrsMLkeIuQ'; - const name = 'cat.jpg'; + const destinationFileName = 'cat.jpg'; const tempFolderPath = getTempFolder(); it('downloads a file into the provided folder when given a valid link', async () => { - const { pathToFile, contentType } = await downloadFile(validDownloadLink, tempFolderPath, name); + const { pathToFile, contentType } = await downloadFile(validDownloadLink, tempFolderPath, destinationFileName); expect(fs.existsSync(pathToFile)).to.equal(true); expect(contentType).to.equal('image/jpeg'); }); @@ -17,7 +17,7 @@ describe('download file test', () => { it('download throws when given an invalid link', async () => { let error; try { - await downloadFile(invalidDownloadLink, tempFolderPath, name); + await downloadFile(invalidDownloadLink, tempFolderPath, destinationFileName); } catch (err) { error = err; } diff --git a/src/utils/temp_folder.test.ts b/src/utils/temp_folder.test.ts index 1ee696a1..2f377b14 100644 --- a/src/utils/temp_folder.test.ts +++ b/src/utils/temp_folder.test.ts @@ -1,17 +1,25 @@ import { expect } from 'chai'; import { cleanUpTempFolder, getTempFolder } from './temp_folder'; import * as fs from 'fs'; +import * as os from 'os'; -describe('temp folder test', () => { - it('temp returns a folder that exists', () => { +describe('getTempFolder function', () => { + it('returns a folder that exists', () => { const tempFolderPath = getTempFolder(); expect(fs.existsSync(tempFolderPath)).to.equal(true); }); - it('temp path returned contains the correct subfolders', () => { + it('getTempFolder can be called twice in a row', () => { const tempFolderPath = getTempFolder(); - expect(tempFolderPath).to.contains('ardrive'); - expect(tempFolderPath).to.contains('temp'); + const tempFolderPath2 = getTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(true); + expect(fs.existsSync(tempFolderPath2)).to.equal(true); + }); + + it('returns a folder that contains the correct subfolders', () => { + const tempFolderPath = getTempFolder(); + const expectedPathComponent = os.platform() === 'win32' ? '\\ardrive-temp' : '/.ardrive/temp'; + expect(tempFolderPath).to.contains(expectedPathComponent); }); it('cleanUpTempFolder removes the temporary folder from the local system', () => { @@ -20,4 +28,12 @@ describe('temp folder test', () => { cleanUpTempFolder(); expect(fs.existsSync(tempFolderPath)).to.equal(false); }); + it('cleanUpTempFolder can be called twice in a row', () => { + const tempFolderPath = getTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(true); + cleanUpTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(false); + cleanUpTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(false); + }); }); From e710f2a58fd027517c3862ab068ac80878a9d23e Mon Sep 17 00:00:00 2001 From: Javed Hussein Date: Wed, 3 Aug 2022 10:28:30 -0400 Subject: [PATCH 37/53] test: add more describes PE-1904 --- src/utils/temp_folder.test.ts | 60 +++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/src/utils/temp_folder.test.ts b/src/utils/temp_folder.test.ts index 2f377b14..e69f5071 100644 --- a/src/utils/temp_folder.test.ts +++ b/src/utils/temp_folder.test.ts @@ -3,37 +3,41 @@ import { cleanUpTempFolder, getTempFolder } from './temp_folder'; import * as fs from 'fs'; import * as os from 'os'; -describe('getTempFolder function', () => { - it('returns a folder that exists', () => { - const tempFolderPath = getTempFolder(); - expect(fs.existsSync(tempFolderPath)).to.equal(true); - }); +describe('temp folder functions', () => { + describe('getTempFolder function', () => { + it('returns a folder that exists', () => { + const tempFolderPath = getTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(true); + }); - it('getTempFolder can be called twice in a row', () => { - const tempFolderPath = getTempFolder(); - const tempFolderPath2 = getTempFolder(); - expect(fs.existsSync(tempFolderPath)).to.equal(true); - expect(fs.existsSync(tempFolderPath2)).to.equal(true); - }); + it('getTempFolder can be called twice in a row', () => { + const tempFolderPath = getTempFolder(); + const tempFolderPath2 = getTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(true); + expect(fs.existsSync(tempFolderPath2)).to.equal(true); + }); - it('returns a folder that contains the correct subfolders', () => { - const tempFolderPath = getTempFolder(); - const expectedPathComponent = os.platform() === 'win32' ? '\\ardrive-temp' : '/.ardrive/temp'; - expect(tempFolderPath).to.contains(expectedPathComponent); + it('returns a folder that contains the correct subfolders', () => { + const tempFolderPath = getTempFolder(); + const expectedPathComponent = os.platform() === 'win32' ? '\\ardrive-temp' : '/.ardrive/temp'; + expect(tempFolderPath).to.contains(expectedPathComponent); + }); }); - it('cleanUpTempFolder removes the temporary folder from the local system', () => { - const tempFolderPath = getTempFolder(); - expect(fs.existsSync(tempFolderPath)).to.equal(true); - cleanUpTempFolder(); - expect(fs.existsSync(tempFolderPath)).to.equal(false); - }); - it('cleanUpTempFolder can be called twice in a row', () => { - const tempFolderPath = getTempFolder(); - expect(fs.existsSync(tempFolderPath)).to.equal(true); - cleanUpTempFolder(); - expect(fs.existsSync(tempFolderPath)).to.equal(false); - cleanUpTempFolder(); - expect(fs.existsSync(tempFolderPath)).to.equal(false); + describe('cleanUpTempFolder function', () => { + it('cleanUpTempFolder removes the temporary folder from the local system', () => { + const tempFolderPath = getTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(true); + cleanUpTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(false); + }); + it('cleanUpTempFolder can be called twice in a row', () => { + const tempFolderPath = getTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(true); + cleanUpTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(false); + cleanUpTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(false); + }); }); }); From a00fe64c295c3acd813c67549c08c983e1a4d972 Mon Sep 17 00:00:00 2001 From: Javed Hussein Date: Wed, 3 Aug 2022 12:21:40 -0400 Subject: [PATCH 38/53] refactor: add cleanup temp to download test PE-1904 --- src/utils/download_file.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/utils/download_file.test.ts b/src/utils/download_file.test.ts index 55a31c78..941e26b4 100644 --- a/src/utils/download_file.test.ts +++ b/src/utils/download_file.test.ts @@ -1,5 +1,5 @@ import { expect } from 'chai'; -import { getTempFolder } from './temp_folder'; +import { cleanUpTempFolder, getTempFolder } from './temp_folder'; import { downloadFile } from './download_file'; import * as fs from 'fs'; @@ -26,4 +26,8 @@ describe('downloadFile function', () => { 'Failed to download file from remote path https://arweave.net/pVoSqZgJUCiNw7oS6CtlVEV8gREQlpRbccrsMLkeIuQ: Request failed with status code 404' ); }); + + after(() => { + cleanUpTempFolder(); + }); }); From 9df05d9ad5a045205bc77ff5872c31e473f4a8c7 Mon Sep 17 00:00:00 2001 From: Javed Hussein Date: Wed, 3 Aug 2022 12:24:13 -0400 Subject: [PATCH 39/53] test: update src/utils/temp_folder.test.ts Co-authored-by: arielmelendez --- src/utils/temp_folder.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/temp_folder.test.ts b/src/utils/temp_folder.test.ts index e69f5071..126d36e8 100644 --- a/src/utils/temp_folder.test.ts +++ b/src/utils/temp_folder.test.ts @@ -31,7 +31,7 @@ describe('temp folder functions', () => { cleanUpTempFolder(); expect(fs.existsSync(tempFolderPath)).to.equal(false); }); - it('cleanUpTempFolder can be called twice in a row', () => { + it('can be called twice in a row', () => { const tempFolderPath = getTempFolder(); expect(fs.existsSync(tempFolderPath)).to.equal(true); cleanUpTempFolder(); From 6fbdf0ec748679e8d49642bcdd0d941bf31e1df8 Mon Sep 17 00:00:00 2001 From: Javed Hussein Date: Wed, 3 Aug 2022 12:24:23 -0400 Subject: [PATCH 40/53] test: update src/utils/temp_folder.test.ts Co-authored-by: arielmelendez --- src/utils/temp_folder.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/utils/temp_folder.test.ts b/src/utils/temp_folder.test.ts index 126d36e8..15ad42a7 100644 --- a/src/utils/temp_folder.test.ts +++ b/src/utils/temp_folder.test.ts @@ -31,6 +31,7 @@ describe('temp folder functions', () => { cleanUpTempFolder(); expect(fs.existsSync(tempFolderPath)).to.equal(false); }); + it('can be called twice in a row', () => { const tempFolderPath = getTempFolder(); expect(fs.existsSync(tempFolderPath)).to.equal(true); From b818931f9b318bdeccf68f815411207e06cf2181 Mon Sep 17 00:00:00 2001 From: Javed Hussein Date: Wed, 3 Aug 2022 12:24:35 -0400 Subject: [PATCH 41/53] test: update src/utils/temp_folder.test.ts Co-authored-by: arielmelendez --- src/utils/temp_folder.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/temp_folder.test.ts b/src/utils/temp_folder.test.ts index 15ad42a7..3e754337 100644 --- a/src/utils/temp_folder.test.ts +++ b/src/utils/temp_folder.test.ts @@ -10,7 +10,7 @@ describe('temp folder functions', () => { expect(fs.existsSync(tempFolderPath)).to.equal(true); }); - it('getTempFolder can be called twice in a row', () => { + it('can be called twice in a row', () => { const tempFolderPath = getTempFolder(); const tempFolderPath2 = getTempFolder(); expect(fs.existsSync(tempFolderPath)).to.equal(true); From 28e904e7d4671279ce7bc6f7983fefc9e949b352 Mon Sep 17 00:00:00 2001 From: Javed Hussein Date: Wed, 3 Aug 2022 12:24:47 -0400 Subject: [PATCH 42/53] test: update src/utils/temp_folder.test.ts Co-authored-by: arielmelendez --- src/utils/temp_folder.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/temp_folder.test.ts b/src/utils/temp_folder.test.ts index 3e754337..c712526d 100644 --- a/src/utils/temp_folder.test.ts +++ b/src/utils/temp_folder.test.ts @@ -25,7 +25,7 @@ describe('temp folder functions', () => { }); describe('cleanUpTempFolder function', () => { - it('cleanUpTempFolder removes the temporary folder from the local system', () => { + it('removes the temporary folder from the local system', () => { const tempFolderPath = getTempFolder(); expect(fs.existsSync(tempFolderPath)).to.equal(true); cleanUpTempFolder(); From eeb1a2800ac87495d83100d978d661025138d8da Mon Sep 17 00:00:00 2001 From: Javed Hussein Date: Wed, 3 Aug 2022 15:20:11 -0400 Subject: [PATCH 43/53] refactor: update src/parameter_declarations.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matías Batista --- src/parameter_declarations.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/parameter_declarations.ts b/src/parameter_declarations.ts index 00fb643a..ce7c66e3 100644 --- a/src/parameter_declarations.ts +++ b/src/parameter_declarations.ts @@ -500,6 +500,9 @@ Parameter.declare({ \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-paths \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-path \t\t\t\t\t\t\t• Can NOT be used in conjunction with --local-csv`, + requiredConjunctionParameters: [ + DestinationFileNameParameter + ], forbiddenConjunctionParameters: [ LocalFilePathParameter_DEPRECATED, LocalPathsParameter, From df8bd8d33c843d8e0fe1d35c962b539fe0b5211a Mon Sep 17 00:00:00 2001 From: Mati Date: Wed, 3 Aug 2022 19:58:11 -0300 Subject: [PATCH 44/53] feat(version): after-release version bump PE-2016 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 37b73378..27cf799e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ardrive-cli", - "version": "1.19.0", + "version": "1.20.0", "description": "The ArDrive Command Line Interface (CLI is a Node.js application for terminal-based ArDrive workflows. It also offers utility operations for securely interacting with Arweave wallets and inspecting various Arweave blockchain conditions.", "main": "./lib/index.js", "bin": { From 53722f458d3864b06be58998394fef8b6ad8252d Mon Sep 17 00:00:00 2001 From: jdaev Date: Tue, 9 Aug 2022 11:59:33 -0400 Subject: [PATCH 45/53] refactor: update temp folder path PE-1904 --- src/utils/temp_folder.test.ts | 2 +- src/utils/temp_folder.ts | 58 ++++++++++++++++++++++++++++++++--- 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/src/utils/temp_folder.test.ts b/src/utils/temp_folder.test.ts index 2f377b14..144511ab 100644 --- a/src/utils/temp_folder.test.ts +++ b/src/utils/temp_folder.test.ts @@ -18,7 +18,7 @@ describe('getTempFolder function', () => { it('returns a folder that contains the correct subfolders', () => { const tempFolderPath = getTempFolder(); - const expectedPathComponent = os.platform() === 'win32' ? '\\ardrive-temp' : '/.ardrive/temp'; + const expectedPathComponent = os.platform() === 'win32' ? '\\ardrive-downloads' : '/.ardrive/ardrive-downloads'; expect(tempFolderPath).to.contains(expectedPathComponent); }); diff --git a/src/utils/temp_folder.ts b/src/utils/temp_folder.ts index 948d0d8f..3b4c5652 100644 --- a/src/utils/temp_folder.ts +++ b/src/utils/temp_folder.ts @@ -2,11 +2,61 @@ import * as fs from 'fs'; import * as os from 'os'; import path from 'path'; +const isWindows = os.platform() === 'win32'; + +function getOrCreateTempBaseFolder(): string { + if (isWindows) { + return getValidWindowsTempPath() ?? getManualWindowsTempPath(); + } else { + return getValidUnixTempPath() ?? getManualUnixTempPath(); + } +} + +function getValidWindowsTempPath(): string | null { + const envTempFolder = process.env['TEMP'] ?? process.env['TMP']; + if (envTempFolder) { + if (fs.existsSync(envTempFolder)) { + return envTempFolder; + } else { + const userProfile = process.env['USERPROFILE']; + if (userProfile) { + const tempPath = path.join(userProfile, 'AppData', 'Local', 'Temp'); + if (fs.existsSync(tempPath)) { + return tempPath; + } + } + } + } + return null; +} + +function getValidUnixTempPath(): string | null { + const envTempFolder = process.env['TMPDIR'] ?? process.env['TMP'] ?? process.env['TEMP']; + if (envTempFolder) { + if (fs.existsSync(envTempFolder)) { + return envTempFolder; + } else { + const tempPath = path.join('tmp'); + if (fs.existsSync(tempPath)) { + return tempPath; + } + } + } + return null; +} + +function getManualWindowsTempPath(): string { + return path.join(os.homedir(), 'ardrive-temp'); +} + +function getManualUnixTempPath(): string { + return path.join(os.homedir(), '.ardrive'); +} + function platformTempFolder(): string { - const tempBaseFolder = process.env['XDG_RUNTIME_DIR'] ?? os.homedir(); - return os.platform() === 'win32' - ? path.join(tempBaseFolder, 'ardrive-temp') - : path.join(tempBaseFolder, '.ardrive', 'temp'); + const tempBaseFolder = getOrCreateTempBaseFolder(); + console.log(tempBaseFolder); + return path.join(tempBaseFolder, 'ardrive-downloads'); } /** * Gets a folder path for storing temporary files. From 099ce72a8a1785bd30de6c3408fdb31cf42a0e4a Mon Sep 17 00:00:00 2001 From: jdaev Date: Tue, 9 Aug 2022 13:18:40 -0400 Subject: [PATCH 46/53] refactor: remove log PE-1904 --- src/utils/temp_folder.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/utils/temp_folder.ts b/src/utils/temp_folder.ts index 3b4c5652..a7cf538c 100644 --- a/src/utils/temp_folder.ts +++ b/src/utils/temp_folder.ts @@ -55,7 +55,6 @@ function getManualUnixTempPath(): string { function platformTempFolder(): string { const tempBaseFolder = getOrCreateTempBaseFolder(); - console.log(tempBaseFolder); return path.join(tempBaseFolder, 'ardrive-downloads'); } /** From e2f77ea2282616d858793ce99d0789771fdcae31 Mon Sep 17 00:00:00 2001 From: jdaev Date: Tue, 9 Aug 2022 13:48:09 -0400 Subject: [PATCH 47/53] fix: fix sourceUri on remote-path PE-2037 --- src/commands/upload_file.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index cb7d0be2..aacf7143 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -48,6 +48,8 @@ interface UploadPathParameter { type FilePath = string; +let remoteUri: string | undefined; + function getFilesFromCSV(parameters: ParametersHelper): UploadPathParameter[] | undefined { const localCSVFile = parameters.getParameterValue(LocalFilesParameter_DEPRECATED) ?? parameters.getParameterValue(LocalCSVParameter); @@ -143,7 +145,7 @@ async function getRemoteFile( parentFolderId: FolderID ): Promise { const remoteFilePath = parameters.getParameterValue(RemotePathParameter); - + remoteUri = remoteFilePath; if (!remoteFilePath) { return undefined; } @@ -272,6 +274,11 @@ new CLICommand({ prompts: fileAndFolderUploadConflictPrompts }); + if (remoteUri) { + // TODO: Include ArFSRemoteFileToUpload functionality in ArDrive Core + results.created[0].sourceUri = remoteUri; + } + console.log(JSON.stringify(results, null, 4)); cleanUpTempFolder(); return SUCCESS_EXIT_CODE; From 9ddb903d990fea56b6b371c686dd2bbb655f352d Mon Sep 17 00:00:00 2001 From: jdaev Date: Tue, 9 Aug 2022 13:53:29 -0400 Subject: [PATCH 48/53] refactor: remove global var PE-2037 --- src/commands/upload_file.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index aacf7143..918f4727 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -48,8 +48,6 @@ interface UploadPathParameter { type FilePath = string; -let remoteUri: string | undefined; - function getFilesFromCSV(parameters: ParametersHelper): UploadPathParameter[] | undefined { const localCSVFile = parameters.getParameterValue(LocalFilesParameter_DEPRECATED) ?? parameters.getParameterValue(LocalCSVParameter); @@ -145,7 +143,6 @@ async function getRemoteFile( parentFolderId: FolderID ): Promise { const remoteFilePath = parameters.getParameterValue(RemotePathParameter); - remoteUri = remoteFilePath; if (!remoteFilePath) { return undefined; } @@ -229,6 +226,7 @@ new CLICommand({ const conflictResolution = parameters.getFileNameConflictResolution(); const shouldBundle = !!parameters.getParameterValue(ShouldBundleParameter); + const remoteFilePath = parameters.getParameterValue(RemotePathParameter); const arweave = getArweaveFromURL(parameters.getGateway()); @@ -274,9 +272,9 @@ new CLICommand({ prompts: fileAndFolderUploadConflictPrompts }); - if (remoteUri) { + if (remoteFilePath) { // TODO: Include ArFSRemoteFileToUpload functionality in ArDrive Core - results.created[0].sourceUri = remoteUri; + results.created[0].sourceUri = remoteFilePath; } console.log(JSON.stringify(results, null, 4)); From e8374b42436a4d75905d693f3353a2f37ff8b131 Mon Sep 17 00:00:00 2001 From: jdaev Date: Tue, 9 Aug 2022 13:57:59 -0400 Subject: [PATCH 49/53] refactor: check shape of result PE-2037 --- src/commands/upload_file.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index 918f4727..9ae94d51 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -272,8 +272,9 @@ new CLICommand({ prompts: fileAndFolderUploadConflictPrompts }); - if (remoteFilePath) { + if (remoteFilePath && results.created[0]) { // TODO: Include ArFSRemoteFileToUpload functionality in ArDrive Core + // TODO: Account for bulk remote path uploads in the future results.created[0].sourceUri = remoteFilePath; } From b3da3006b200392dbe6c4542e105cab86bdab1a7 Mon Sep 17 00:00:00 2001 From: jdaev Date: Tue, 9 Aug 2022 14:02:33 -0400 Subject: [PATCH 50/53] refactor: update result check PE-2037 --- src/commands/upload_file.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/upload_file.ts b/src/commands/upload_file.ts index 9ae94d51..62ae3b7e 100644 --- a/src/commands/upload_file.ts +++ b/src/commands/upload_file.ts @@ -272,7 +272,7 @@ new CLICommand({ prompts: fileAndFolderUploadConflictPrompts }); - if (remoteFilePath && results.created[0]) { + if (remoteFilePath && results.created[0].type === 'file') { // TODO: Include ArFSRemoteFileToUpload functionality in ArDrive Core // TODO: Account for bulk remote path uploads in the future results.created[0].sourceUri = remoteFilePath; From db0407d7d363020f1e3a9dce14e7c41bf801826b Mon Sep 17 00:00:00 2001 From: jdaev Date: Wed, 10 Aug 2022 09:17:25 -0400 Subject: [PATCH 51/53] refactor: remove path.join PE-1904 --- src/utils/temp_folder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/temp_folder.ts b/src/utils/temp_folder.ts index a7cf538c..883f1976 100644 --- a/src/utils/temp_folder.ts +++ b/src/utils/temp_folder.ts @@ -36,7 +36,7 @@ function getValidUnixTempPath(): string | null { if (fs.existsSync(envTempFolder)) { return envTempFolder; } else { - const tempPath = path.join('tmp'); + const tempPath = 'tmp'; if (fs.existsSync(tempPath)) { return tempPath; } From 7340cd3f8705f041fa0b7bc9b5e9f28befc9b0aa Mon Sep 17 00:00:00 2001 From: jdaev Date: Wed, 10 Aug 2022 09:19:56 -0400 Subject: [PATCH 52/53] refactor: correct temp path PE-1904 --- src/utils/temp_folder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/temp_folder.ts b/src/utils/temp_folder.ts index 883f1976..ad9e0c58 100644 --- a/src/utils/temp_folder.ts +++ b/src/utils/temp_folder.ts @@ -36,7 +36,7 @@ function getValidUnixTempPath(): string | null { if (fs.existsSync(envTempFolder)) { return envTempFolder; } else { - const tempPath = 'tmp'; + const tempPath = '/tmp'; if (fs.existsSync(tempPath)) { return tempPath; } From 9364392aad01a2134904bb40dcd2c82e05065c83 Mon Sep 17 00:00:00 2001 From: Javed Hussein Date: Wed, 10 Aug 2022 20:07:51 -0400 Subject: [PATCH 53/53] test: add back test describes PE-1904 --- src/utils/temp_folder.test.ts | 61 +++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/src/utils/temp_folder.test.ts b/src/utils/temp_folder.test.ts index 144511ab..069b6516 100644 --- a/src/utils/temp_folder.test.ts +++ b/src/utils/temp_folder.test.ts @@ -3,37 +3,42 @@ import { cleanUpTempFolder, getTempFolder } from './temp_folder'; import * as fs from 'fs'; import * as os from 'os'; -describe('getTempFolder function', () => { - it('returns a folder that exists', () => { - const tempFolderPath = getTempFolder(); - expect(fs.existsSync(tempFolderPath)).to.equal(true); - }); +describe('temp folder functions', () => { + describe('getTempFolder function', () => { + it('returns a folder that exists', () => { + const tempFolderPath = getTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(true); + }); - it('getTempFolder can be called twice in a row', () => { - const tempFolderPath = getTempFolder(); - const tempFolderPath2 = getTempFolder(); - expect(fs.existsSync(tempFolderPath)).to.equal(true); - expect(fs.existsSync(tempFolderPath2)).to.equal(true); - }); + it('getTempFolder can be called twice in a row', () => { + const tempFolderPath = getTempFolder(); + const tempFolderPath2 = getTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(true); + expect(fs.existsSync(tempFolderPath2)).to.equal(true); + }); - it('returns a folder that contains the correct subfolders', () => { - const tempFolderPath = getTempFolder(); - const expectedPathComponent = os.platform() === 'win32' ? '\\ardrive-downloads' : '/.ardrive/ardrive-downloads'; - expect(tempFolderPath).to.contains(expectedPathComponent); + it('returns a folder that contains the correct subfolders', () => { + const tempFolderPath = getTempFolder(); + const expectedPathComponent = + os.platform() === 'win32' ? '\\ardrive-downloads' : '/.ardrive/ardrive-downloads'; + expect(tempFolderPath).to.contains(expectedPathComponent); + }); }); - it('cleanUpTempFolder removes the temporary folder from the local system', () => { - const tempFolderPath = getTempFolder(); - expect(fs.existsSync(tempFolderPath)).to.equal(true); - cleanUpTempFolder(); - expect(fs.existsSync(tempFolderPath)).to.equal(false); - }); - it('cleanUpTempFolder can be called twice in a row', () => { - const tempFolderPath = getTempFolder(); - expect(fs.existsSync(tempFolderPath)).to.equal(true); - cleanUpTempFolder(); - expect(fs.existsSync(tempFolderPath)).to.equal(false); - cleanUpTempFolder(); - expect(fs.existsSync(tempFolderPath)).to.equal(false); + describe('cleanUpTempFolder function', () => { + it('cleanUpTempFolder removes the temporary folder from the local system', () => { + const tempFolderPath = getTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(true); + cleanUpTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(false); + }); + it('cleanUpTempFolder can be called twice in a row', () => { + const tempFolderPath = getTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(true); + cleanUpTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(false); + cleanUpTempFolder(); + expect(fs.existsSync(tempFolderPath)).to.equal(false); + }); }); });