From bb397cbc64382efb9d8bb3c230a2e901dfe5273f Mon Sep 17 00:00:00 2001 From: Jackson Weber Date: Sun, 28 Jan 2024 20:36:53 -0800 Subject: [PATCH 1/9] Add process ID to created file name for logs caching. --- Library/Sender.ts | 7 ++-- package-lock.json | 83 ++++++++++++++++++++++++++++------------------- 2 files changed, 53 insertions(+), 37 deletions(-) diff --git a/Library/Sender.ts b/Library/Sender.ts index 00494e1d..5445b0c6 100644 --- a/Library/Sender.ts +++ b/Library/Sender.ts @@ -48,6 +48,7 @@ class Sender { private _fileCleanupTimer: NodeJS.Timer; private _redirectedHost: string = null; private _tempDir: string; + private _processId: number = process.pid; private _requestTimedOut: boolean; protected _resendInterval: number; protected _maxBytesOnDisk: number; @@ -428,7 +429,7 @@ class Sender { try { //create file - file name for now is the timestamp, a better approach would be a UUID but that //would require an external dependency - var fileName = new Date().getTime() + ".ai.json"; + var fileName = `${new Date().getTime()}.${this._processId}.ai.json`; var fileFullPath = path.join(this._tempDir, fileName); // Mode 600 is w/r for creator and no read access for others (only applies on *nix) @@ -465,7 +466,7 @@ class Sender { //create file - file name for now is the timestamp, a better approach would be a UUID but that //would require an external dependency - var fileName = new Date().getTime() + ".ai.json"; + var fileName = `${new Date().getTime()}.${this._processId}.ai.json`; var fileFullPath = path.join(this._tempDir, fileName); // Mode 600 is w/r for creator and no access for anyone else (only applies on *nix) @@ -514,7 +515,7 @@ class Sender { if (files.length > 0) { for (let i = 0; i < files.length; i++) { // Check expiration - let fileCreationDate: Date = new Date(parseInt(files[i].split(".ai.json")[0])); + let fileCreationDate: Date = new Date(parseInt(files[i].split(`${this._processId}.ai.json`)[0])); let expired = new Date(+(new Date()) - Sender.FILE_RETEMPTION_PERIOD) > fileCreationDate; if (expired) { var filePath = path.join(this._tempDir, files[i]); diff --git a/package-lock.json b/package-lock.json index 405327d8..237f9c45 100644 --- a/package-lock.json +++ b/package-lock.json @@ -222,13 +222,13 @@ } }, "node_modules/@humanwhocodes/config-array": { - "version": "0.11.13", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.13.tgz", - "integrity": "sha512-JSBDMiDKSzQVngfRjOdFXgFfklaXI4K9nLF49Auh21lmBWRLIK3+xTErTWD4KU54pb6coM6ESE7Awz/FNU3zgQ==", + "version": "0.11.14", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", + "integrity": "sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==", "dev": true, "dependencies": { - "@humanwhocodes/object-schema": "^2.0.1", - "debug": "^4.1.1", + "@humanwhocodes/object-schema": "^2.0.2", + "debug": "^4.3.1", "minimatch": "^3.0.5" }, "engines": { @@ -249,9 +249,9 @@ } }, "node_modules/@humanwhocodes/object-schema": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.1.tgz", - "integrity": "sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.2.tgz", + "integrity": "sha512-6EwiSjwWYP7pTckG6I5eyFANjPhmPjUX9JRLUSfNPC7FX7zK9gyZAfUEaECL6ALTpGX5AjnBq3C9XmVWPitNpw==", "dev": true }, "node_modules/@mapbox/node-pre-gyp": { @@ -323,11 +323,11 @@ } }, "node_modules/@opentelemetry/core": { - "version": "1.19.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.19.0.tgz", - "integrity": "sha512-w42AukJh3TP8R0IZZOVJVM/kMWu8g+lm4LzT70WtuKqhwq7KVhcDzZZuZinWZa6TtQCl7Smt2wolEYzpHabOgw==", + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.21.0.tgz", + "integrity": "sha512-KP+OIweb3wYoP7qTYL/j5IpOlu52uxBv5M4+QhSmmUfLyTgu1OIS71msK3chFo1D6Y61BIH3wMiMYRCxJCQctA==", "dependencies": { - "@opentelemetry/semantic-conventions": "1.19.0" + "@opentelemetry/semantic-conventions": "1.21.0" }, "engines": { "node": ">=14" @@ -355,12 +355,12 @@ } }, "node_modules/@opentelemetry/resources": { - "version": "1.19.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.19.0.tgz", - "integrity": "sha512-RgxvKuuMOf7nctOeOvpDjt2BpZvZGr9Y0vf7eGtY5XYZPkh2p7e2qub1S2IArdBMf9kEbz0SfycqCviOu9isqg==", + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.21.0.tgz", + "integrity": "sha512-1Z86FUxPKL6zWVy2LdhueEGl9AHDJcx+bvHStxomruz6Whd02mE3lNUMjVJ+FGRoktx/xYQcxccYb03DiUP6Yw==", "dependencies": { - "@opentelemetry/core": "1.19.0", - "@opentelemetry/semantic-conventions": "1.19.0" + "@opentelemetry/core": "1.21.0", + "@opentelemetry/semantic-conventions": "1.21.0" }, "engines": { "node": ">=14" @@ -370,13 +370,13 @@ } }, "node_modules/@opentelemetry/sdk-trace-base": { - "version": "1.19.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.19.0.tgz", - "integrity": "sha512-+IRvUm+huJn2KqfFW3yW/cjvRwJ8Q7FzYHoUNx5Fr0Lws0LxjMJG1uVB8HDpLwm7mg5XXH2M5MF+0jj5cM8BpQ==", + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.21.0.tgz", + "integrity": "sha512-yrElGX5Fv0umzp8Nxpta/XqU71+jCAyaLk34GmBzNcrW43nqbrqvdPs4gj4MVy/HcTjr6hifCDCYA3rMkajxxA==", "dependencies": { - "@opentelemetry/core": "1.19.0", - "@opentelemetry/resources": "1.19.0", - "@opentelemetry/semantic-conventions": "1.19.0" + "@opentelemetry/core": "1.21.0", + "@opentelemetry/resources": "1.21.0", + "@opentelemetry/semantic-conventions": "1.21.0" }, "engines": { "node": ">=14" @@ -386,9 +386,9 @@ } }, "node_modules/@opentelemetry/semantic-conventions": { - "version": "1.19.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.19.0.tgz", - "integrity": "sha512-14jRpC8f5c0gPSwoZ7SbEJni1PqI+AhAE8m1bMz6v+RPM4OlP1PT2UHBJj5Qh/ALLPjhVU/aZUK3YyjTUqqQVg==", + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.21.0.tgz", + "integrity": "sha512-lkC8kZYntxVKr7b8xmjCVUgE0a8xgDakPyDo9uSWavXPyYqLgYYGdEd2j8NxihRyb6UwpX3G/hFUF4/9q2V+/g==", "engines": { "node": ">=14" } @@ -1268,9 +1268,9 @@ "dev": true }, "node_modules/fastq": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.16.0.tgz", - "integrity": "sha512-ifCoaXsDrsdkWTtiNJX5uzHDsrck5TzfKKDcuFFTIrrc/BS076qgEIfoIy1VeZqViznfKiysPYTh/QeHtnIsYA==", + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.0.tgz", + "integrity": "sha512-zGygtijUMT7jnk3h26kUms3BkSDp4IfIKjmnqI2tvx6nuBfiF1UqOxbnLfzdv+apBy+53oaImsKtMw/xYbW+1w==", "dev": true, "dependencies": { "reusify": "^1.0.4" @@ -1407,6 +1407,20 @@ "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", "dev": true }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", @@ -2668,15 +2682,16 @@ "dev": true }, "node_modules/set-function-length": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.1.1.tgz", - "integrity": "sha512-VoaqjbBJKiWtg4yRcKBQ7g7wnGnLV3M8oLvVWwOk2PdYY6PEFegR1vezXR0tw6fZGF9csVakIRjrJiy2veSBFQ==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.0.tgz", + "integrity": "sha512-4DBHDoyHlM1IRPGYcoxexgh67y4ueR53FKV1yyxwFMY7aCqcN/38M1+SwZ/qJQ8iLv7+ck385ot4CcisOAPT9w==", "dev": true, "dependencies": { "define-data-property": "^1.1.1", - "get-intrinsic": "^1.2.1", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.2", "gopd": "^1.0.1", - "has-property-descriptors": "^1.0.0" + "has-property-descriptors": "^1.0.1" }, "engines": { "node": ">= 0.4" From cf395608cd5776808ae15bfdf5c23a976cdd9a06 Mon Sep 17 00:00:00 2001 From: Jackson Weber Date: Sun, 28 Jan 2024 22:01:13 -0800 Subject: [PATCH 2/9] Test --- Library/Sender.ts | 6 +++++- Tests/EndToEnd.tests.ts | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Library/Sender.ts b/Library/Sender.ts index 5445b0c6..c8cb6ffc 100644 --- a/Library/Sender.ts +++ b/Library/Sender.ts @@ -431,6 +431,7 @@ class Sender { //would require an external dependency var fileName = `${new Date().getTime()}.${this._processId}.ai.json`; var fileFullPath = path.join(this._tempDir, fileName); + console.log("PRINTING THE FILE PATH CREATED: ", fileFullPath); // Mode 600 is w/r for creator and no read access for others (only applies on *nix) // For Windows, ACL rules are applied to the entire directory (see logic in _confirmDirExists and _applyACLRules) @@ -486,10 +487,13 @@ class Sender { private async _sendFirstFileOnDisk(): Promise { try { let files = await FileSystemHelper.readdirAsync(this._tempDir); - files = files.filter(f => path.basename(f).indexOf(".ai.json") > -1); + console.log("FILES IN THE TEMP DIR: ", files); + files = files.filter(f => path.basename(f).indexOf(`.${process.pid}.ai.json`) > -1); + console.log("STANDARD TEST PID: ", this._processId) if (files.length > 0) { var firstFile = files[0]; var filePath = path.join(this._tempDir, firstFile); + // console.log("READING THE FIRST FILE ON DISK: ", filePath); let buffer = await FileSystemHelper.readFileAsync(filePath); // delete the file first to prevent double sending await FileSystemHelper.unlinkAsync(filePath); diff --git a/Tests/EndToEnd.tests.ts b/Tests/EndToEnd.tests.ts index 7f0a1a59..789dd049 100644 --- a/Tests/EndToEnd.tests.ts +++ b/Tests/EndToEnd.tests.ts @@ -23,7 +23,7 @@ import { JsonConfig } from "../Library/JsonConfig"; import { FileAccessControl } from "../Library/FileAccessControl"; import FileSystemHelper = require("../Library/FileSystemHelper"); import AutoCollectHttpRequests = require("../AutoCollection/HttpRequests"); - +console.log("E2E PID: ", process.pid); /** * A fake response class that passes by default */ @@ -827,11 +827,11 @@ describe("EndToEnd", () => { callback: (response: any) => { // wait until sdk looks for offline files setTimeout(() => { - assert.equal(readdir.callCount, 2); + assert.equal(readdir.callCount, 2, readdir.callCount.toString()); assert.equal(readFile.callCount, 1); assert.equal( path.dirname(readFile.firstCall.args[0]), - path.join(os.tmpdir(), Sender.TEMPDIR_PREFIX + "key")); + path.join(os.tmpdir(), Sender.TEMPDIR_PREFIX + "key"), "oopsie"); done(); }, 100); } From 8c61ae4bbdef309cce36ae6361615034b36bbda5 Mon Sep 17 00:00:00 2001 From: Jackson Weber Date: Sun, 28 Jan 2024 22:19:13 -0800 Subject: [PATCH 3/9] Update file cleanup to only clean files linked to the PID. --- Library/Sender.ts | 8 ++------ Tests/EndToEnd.tests.ts | 7 +++---- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/Library/Sender.ts b/Library/Sender.ts index c8cb6ffc..0ffae029 100644 --- a/Library/Sender.ts +++ b/Library/Sender.ts @@ -431,7 +431,6 @@ class Sender { //would require an external dependency var fileName = `${new Date().getTime()}.${this._processId}.ai.json`; var fileFullPath = path.join(this._tempDir, fileName); - console.log("PRINTING THE FILE PATH CREATED: ", fileFullPath); // Mode 600 is w/r for creator and no read access for others (only applies on *nix) // For Windows, ACL rules are applied to the entire directory (see logic in _confirmDirExists and _applyACLRules) @@ -487,13 +486,10 @@ class Sender { private async _sendFirstFileOnDisk(): Promise { try { let files = await FileSystemHelper.readdirAsync(this._tempDir); - console.log("FILES IN THE TEMP DIR: ", files); - files = files.filter(f => path.basename(f).indexOf(`.${process.pid}.ai.json`) > -1); - console.log("STANDARD TEST PID: ", this._processId) + files = files.filter(f => path.basename(f).indexOf(`${process.pid}.ai.json`) > -1); if (files.length > 0) { var firstFile = files[0]; var filePath = path.join(this._tempDir, firstFile); - // console.log("READING THE FIRST FILE ON DISK: ", filePath); let buffer = await FileSystemHelper.readFileAsync(filePath); // delete the file first to prevent double sending await FileSystemHelper.unlinkAsync(filePath); @@ -515,7 +511,7 @@ class Sender { private async _fileCleanupTask(): Promise { try { let files = await FileSystemHelper.readdirAsync(this._tempDir); - files = files.filter(f => path.basename(f).indexOf(".ai.json") > -1); + files = files.filter(f => path.basename(f).indexOf(`${this._processId}.ai.json`) > -1); if (files.length > 0) { for (let i = 0; i < files.length; i++) { // Check expiration diff --git a/Tests/EndToEnd.tests.ts b/Tests/EndToEnd.tests.ts index 789dd049..76f218dc 100644 --- a/Tests/EndToEnd.tests.ts +++ b/Tests/EndToEnd.tests.ts @@ -23,7 +23,6 @@ import { JsonConfig } from "../Library/JsonConfig"; import { FileAccessControl } from "../Library/FileAccessControl"; import FileSystemHelper = require("../Library/FileSystemHelper"); import AutoCollectHttpRequests = require("../AutoCollection/HttpRequests"); -console.log("E2E PID: ", process.pid); /** * A fake response class that passes by default */ @@ -484,8 +483,8 @@ describe("EndToEnd", () => { writeFile = sandbox.stub(FileSystemHelper, 'writeFileAsync'); writeFileSync = sandbox.stub(fs, 'writeFileSync'); existsSync = sandbox.stub(fs, 'existsSync').returns(true); - readdir = sandbox.stub(FileSystemHelper, 'readdirAsync').returns(['1.ai.json']); - readdirSync = sandbox.stub(fs, 'readdirSync').returns(['1.ai.json']); + readdir = sandbox.stub(FileSystemHelper, 'readdirAsync').returns([`${process.pid}.ai.json`]); + readdirSync = sandbox.stub(fs, 'readdirSync').returns([`${process.pid}.ai.json`]); stat = sandbox.stub(FileSystemHelper, 'statAsync').returns({ isFile: () => true, size: 8000 }); statSync = sandbox.stub(fs, 'statSync').returns({ isFile: () => true, size: 8000 }); lstat = sandbox.stub(FileSystemHelper, 'lstatAsync').returns({ isDirectory: () => true }); @@ -827,7 +826,7 @@ describe("EndToEnd", () => { callback: (response: any) => { // wait until sdk looks for offline files setTimeout(() => { - assert.equal(readdir.callCount, 2, readdir.callCount.toString()); + assert.equal(readdir.callCount, 2); assert.equal(readFile.callCount, 1); assert.equal( path.dirname(readFile.firstCall.args[0]), From b9c29dd8e3dbb214d75be736ef4f60df83ef1a6a Mon Sep 17 00:00:00 2001 From: Jackson Weber Date: Sun, 28 Jan 2024 22:26:53 -0800 Subject: [PATCH 4/9] Update EndToEnd.tests.ts --- Tests/EndToEnd.tests.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/EndToEnd.tests.ts b/Tests/EndToEnd.tests.ts index 76f218dc..383c4566 100644 --- a/Tests/EndToEnd.tests.ts +++ b/Tests/EndToEnd.tests.ts @@ -830,7 +830,7 @@ describe("EndToEnd", () => { assert.equal(readFile.callCount, 1); assert.equal( path.dirname(readFile.firstCall.args[0]), - path.join(os.tmpdir(), Sender.TEMPDIR_PREFIX + "key"), "oopsie"); + path.join(os.tmpdir(), Sender.TEMPDIR_PREFIX + "key")); done(); }, 100); } From ebf4dd4ac31a57e8ddd79a5d88056508e94f7171 Mon Sep 17 00:00:00 2001 From: Jackson Weber Date: Mon, 29 Jan 2024 16:23:38 -0800 Subject: [PATCH 5/9] Update error handling in the file system helper. --- Library/FileSystemHelper.ts | 46 +++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/Library/FileSystemHelper.ts b/Library/FileSystemHelper.ts index 8277fbba..6a6346ee 100644 --- a/Library/FileSystemHelper.ts +++ b/Library/FileSystemHelper.ts @@ -1,6 +1,8 @@ import * as fs from "fs"; import path = require("path"); import { promisify } from "util"; +import Logging = require("./Logging"); +import Util = require("./Util"); export const statAsync = promisify(fs.stat); export const lstatAsync = promisify(fs.lstat); @@ -39,38 +41,52 @@ export const confirmDirExists = async (directory: string): Promise => { * Computes the size (in bytes) of all files in a directory at the root level. Asynchronously. */ export const getShallowDirectorySize = async (directory: string): Promise => { - // Get the directory listing - const files = await readdirAsync(directory); let totalSize = 0; - // Query all file sizes - for (const file of files) { - const fileStats = await statAsync(path.join(directory, file)); - if (fileStats.isFile()) { - totalSize += fileStats.size; + try { + // Get the directory listing + const files = await readdirAsync(directory); + // Query all file sizes + for (const file of files) { + const fileStats = await statAsync(path.join(directory, file)); + if (fileStats.isFile()) { + totalSize += fileStats.size; + } } + return totalSize; + } catch (err) { + Logging.warn(`Failed to get directory size for ${directory}: ${Util.dumpObj(err)}`); + return totalSize; } - return totalSize; }; /** * Computes the size (in bytes) of all files in a directory at the root level. Synchronously. */ export const getShallowDirectorySizeSync = (directory: string): number => { - let files = fs.readdirSync(directory); let totalSize = 0; - for (let i = 0; i < files.length; i++) { - totalSize += fs.statSync(path.join(directory, files[i])).size; + try { + let files = fs.readdirSync(directory); + for (let i = 0; i < files.length; i++) { + totalSize += fs.statSync(path.join(directory, files[i])).size; + } + return totalSize; + } catch (err) { + Logging.warn(`Failed to get directory size synchronously for ${directory}: ${Util.dumpObj(err)}`) + return totalSize; } - return totalSize; } /** * Computes the size (in bytes) of a file asynchronously. */ export const getShallowFileSize = async (filePath: string): Promise => { - const fileStats = await statAsync(filePath); - if (fileStats.isFile()) { - return fileStats.size; + try { + const fileStats = await statAsync(filePath); + if (fileStats.isFile()) { + return fileStats.size; + } + } catch (err) { + Logging.warn(`Failed to get file size for ${filePath}: ${Util.dumpObj(err)}`); } } From 00c10d17902ca99e1c7f9a94cb0128899ffd89d0 Mon Sep 17 00:00:00 2001 From: Jackson Weber Date: Mon, 29 Jan 2024 22:05:38 -0800 Subject: [PATCH 6/9] Update FileSystemHelper.ts --- Library/FileSystemHelper.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Library/FileSystemHelper.ts b/Library/FileSystemHelper.ts index 6a6346ee..0a4e28bd 100644 --- a/Library/FileSystemHelper.ts +++ b/Library/FileSystemHelper.ts @@ -2,7 +2,6 @@ import * as fs from "fs"; import path = require("path"); import { promisify } from "util"; import Logging = require("./Logging"); -import Util = require("./Util"); export const statAsync = promisify(fs.stat); export const lstatAsync = promisify(fs.lstat); @@ -53,8 +52,8 @@ export const getShallowDirectorySize = async (directory: string): Promise { totalSize += fs.statSync(path.join(directory, files[i])).size; } return totalSize; - } catch (err) { - Logging.warn(`Failed to get directory size synchronously for ${directory}: ${Util.dumpObj(err)}`) + } catch { + Logging.warn(`Failed to get directory size synchronously for ${directory}`) return totalSize; } } @@ -85,8 +84,8 @@ export const getShallowFileSize = async (filePath: string): Promise => { if (fileStats.isFile()) { return fileStats.size; } - } catch (err) { - Logging.warn(`Failed to get file size for ${filePath}: ${Util.dumpObj(err)}`); + } catch { + Logging.warn(`Failed to get file size for ${filePath}`); } } From 935ca1481bb60e53bc6ba189380d8423c6ca4791 Mon Sep 17 00:00:00 2001 From: Jackson Weber Date: Thu, 1 Feb 2024 13:05:20 -0800 Subject: [PATCH 7/9] Address PR comments. --- Library/FileSystemHelper.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Library/FileSystemHelper.ts b/Library/FileSystemHelper.ts index 0a4e28bd..e16b7707 100644 --- a/Library/FileSystemHelper.ts +++ b/Library/FileSystemHelper.ts @@ -51,11 +51,10 @@ export const getShallowDirectorySize = async (directory: string): Promise { for (let i = 0; i < files.length; i++) { totalSize += fs.statSync(path.join(directory, files[i])).size; } - return totalSize; } catch { Logging.warn(`Failed to get directory size synchronously for ${directory}`) - return totalSize; } + return totalSize } /** -* Computes the size (in bytes) of a file asynchronously. +* Computes the size (in bytes) of a file asynchronously. Returns -1 if the file does not exist. */ export const getShallowFileSize = async (filePath: string): Promise => { try { @@ -86,6 +84,7 @@ export const getShallowFileSize = async (filePath: string): Promise => { } } catch { Logging.warn(`Failed to get file size for ${filePath}`); + return -1; } } From 3f969ef5f69051d90174bdd0d19568417f8924bb Mon Sep 17 00:00:00 2001 From: Jackson Weber Date: Fri, 2 Feb 2024 14:23:40 -0800 Subject: [PATCH 8/9] Remove PID. --- Library/Sender.ts | 11 +++--- Tests/EndToEnd.tests.ts | 4 +-- package-lock.json | 80 ++++++++++++++++++++++++++--------------- 3 files changed, 58 insertions(+), 37 deletions(-) diff --git a/Library/Sender.ts b/Library/Sender.ts index 0ffae029..d32db574 100644 --- a/Library/Sender.ts +++ b/Library/Sender.ts @@ -48,7 +48,6 @@ class Sender { private _fileCleanupTimer: NodeJS.Timer; private _redirectedHost: string = null; private _tempDir: string; - private _processId: number = process.pid; private _requestTimedOut: boolean; protected _resendInterval: number; protected _maxBytesOnDisk: number; @@ -429,7 +428,7 @@ class Sender { try { //create file - file name for now is the timestamp, a better approach would be a UUID but that //would require an external dependency - var fileName = `${new Date().getTime()}.${this._processId}.ai.json`; + var fileName = `${new Date().getTime()}.ai.json`; var fileFullPath = path.join(this._tempDir, fileName); // Mode 600 is w/r for creator and no read access for others (only applies on *nix) @@ -466,7 +465,7 @@ class Sender { //create file - file name for now is the timestamp, a better approach would be a UUID but that //would require an external dependency - var fileName = `${new Date().getTime()}.${this._processId}.ai.json`; + var fileName = `${new Date().getTime()}.ai.json`; var fileFullPath = path.join(this._tempDir, fileName); // Mode 600 is w/r for creator and no access for anyone else (only applies on *nix) @@ -486,7 +485,7 @@ class Sender { private async _sendFirstFileOnDisk(): Promise { try { let files = await FileSystemHelper.readdirAsync(this._tempDir); - files = files.filter(f => path.basename(f).indexOf(`${process.pid}.ai.json`) > -1); + files = files.filter(f => path.basename(f).indexOf(".ai.json") > -1); if (files.length > 0) { var firstFile = files[0]; var filePath = path.join(this._tempDir, firstFile); @@ -511,11 +510,11 @@ class Sender { private async _fileCleanupTask(): Promise { try { let files = await FileSystemHelper.readdirAsync(this._tempDir); - files = files.filter(f => path.basename(f).indexOf(`${this._processId}.ai.json`) > -1); + files = files.filter(f => path.basename(f).indexOf(".ai.json") > -1); if (files.length > 0) { for (let i = 0; i < files.length; i++) { // Check expiration - let fileCreationDate: Date = new Date(parseInt(files[i].split(`${this._processId}.ai.json`)[0])); + let fileCreationDate: Date = new Date(parseInt(files[i].split(".ai.json")[0])); let expired = new Date(+(new Date()) - Sender.FILE_RETEMPTION_PERIOD) > fileCreationDate; if (expired) { var filePath = path.join(this._tempDir, files[i]); diff --git a/Tests/EndToEnd.tests.ts b/Tests/EndToEnd.tests.ts index 383c4566..a51b6571 100644 --- a/Tests/EndToEnd.tests.ts +++ b/Tests/EndToEnd.tests.ts @@ -483,8 +483,8 @@ describe("EndToEnd", () => { writeFile = sandbox.stub(FileSystemHelper, 'writeFileAsync'); writeFileSync = sandbox.stub(fs, 'writeFileSync'); existsSync = sandbox.stub(fs, 'existsSync').returns(true); - readdir = sandbox.stub(FileSystemHelper, 'readdirAsync').returns([`${process.pid}.ai.json`]); - readdirSync = sandbox.stub(fs, 'readdirSync').returns([`${process.pid}.ai.json`]); + readdir = sandbox.stub(FileSystemHelper, 'readdirAsync').returns(['1.ai.json']); + readdirSync = sandbox.stub(fs, 'readdirSync').returns(['.ai.json']); stat = sandbox.stub(FileSystemHelper, 'statAsync').returns({ isFile: () => true, size: 8000 }); statSync = sandbox.stub(fs, 'statSync').returns({ isFile: () => true, size: 8000 }); lstat = sandbox.stub(FileSystemHelper, 'lstatAsync').returns({ isDirectory: () => true }); diff --git a/package-lock.json b/package-lock.json index 237f9c45..c3e99d6e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -61,27 +61,27 @@ } }, "node_modules/@azure/abort-controller": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-1.1.0.tgz", - "integrity": "sha512-TrRLIoSQVzfAJX9H1JeFjzAoDGcoK1IYX1UImfceTZpsyYfWr09Ss1aHW1y5TrrR3iq6RZLBwJ3E24uwPhwahw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.0.0.tgz", + "integrity": "sha512-RP/mR/WJchR+g+nQFJGOec+nzeN/VvjlwbinccoqfhTsTHbb8X5+mLDp48kHT0ueyum0BNSwGm0kX0UZuIqTGg==", "dependencies": { "tslib": "^2.2.0" }, "engines": { - "node": ">=12.0.0" + "node": ">=18.0.0" } }, "node_modules/@azure/core-auth": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.5.0.tgz", - "integrity": "sha512-udzoBuYG1VBoHVohDTrvKjyzel34zt77Bhp7dQntVGGD0ehVq48owENbBG8fIgkHRNUBQH5k1r0hpoMu5L8+kw==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.6.0.tgz", + "integrity": "sha512-3X9wzaaGgRaBCwhLQZDtFp5uLIXCPrGbwJNWPPugvL4xbIGgScv77YzzxToKGLAKvG9amDoofMoP+9hsH1vs1w==", "dependencies": { - "@azure/abort-controller": "^1.0.0", + "@azure/abort-controller": "^2.0.0", "@azure/core-util": "^1.1.0", "tslib": "^2.2.0" }, "engines": { - "node": ">=14.0.0" + "node": ">=18.0.0" } }, "node_modules/@azure/core-rest-pipeline": { @@ -104,6 +104,17 @@ "node": ">=14.0.0" } }, + "node_modules/@azure/core-rest-pipeline/node_modules/@azure/abort-controller": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-1.1.0.tgz", + "integrity": "sha512-TrRLIoSQVzfAJX9H1JeFjzAoDGcoK1IYX1UImfceTZpsyYfWr09Ss1aHW1y5TrrR3iq6RZLBwJ3E24uwPhwahw==", + "dependencies": { + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/@azure/core-tracing": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.0.1.tgz", @@ -127,6 +138,17 @@ "node": ">=14.0.0" } }, + "node_modules/@azure/core-util/node_modules/@azure/abort-controller": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-1.1.0.tgz", + "integrity": "sha512-TrRLIoSQVzfAJX9H1JeFjzAoDGcoK1IYX1UImfceTZpsyYfWr09Ss1aHW1y5TrrR3iq6RZLBwJ3E24uwPhwahw==", + "dependencies": { + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/@azure/functions": { "version": "3.5.1", "resolved": "https://registry.npmjs.org/@azure/functions/-/functions-3.5.1.tgz", @@ -629,9 +651,9 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "node_modules/available-typed-arrays": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", - "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==", + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.6.tgz", + "integrity": "sha512-j1QzY8iPNPG4o4xmO3ptzpRxTciqD3MgEHtifP/YnJpIo58Xu+ne4BejlbkuaLfXn/nz6HFiw29bLpj2PNMdGg==", "dev": true, "engines": { "node": ">= 0.4" @@ -1584,12 +1606,12 @@ } }, "node_modules/has-tostringtag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", - "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", "dev": true, "dependencies": { - "has-symbols": "^1.0.2" + "has-symbols": "^1.0.3" }, "engines": { "node": ">= 0.4" @@ -1662,9 +1684,9 @@ } }, "node_modules/ignore": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.0.tgz", - "integrity": "sha512-g7dmpshy+gD7mh88OC9NwSGTKoc3kyLAZQRU1mt53Aw/vnvfXnbC+F/7F7QoYVKbV+KNvJx8wArewKy1vXMtlg==", + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", + "integrity": "sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==", "dev": true, "engines": { "node": ">= 4" @@ -1846,12 +1868,12 @@ } }, "node_modules/is-typed-array": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.12.tgz", - "integrity": "sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==", + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.13.tgz", + "integrity": "sha512-uZ25/bUAlUY5fR4OKT4rZQEBrzQWYV9ZJYGGsUmEJ6thodVJ1HX64ePQ6Z0qPWP+m+Uq6e9UugrE38jeYsDSMw==", "dev": true, "dependencies": { - "which-typed-array": "^1.1.11" + "which-typed-array": "^1.1.14" }, "engines": { "node": ">= 0.4" @@ -2983,16 +3005,16 @@ } }, "node_modules/which-typed-array": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.13.tgz", - "integrity": "sha512-P5Nra0qjSncduVPEAr7xhoF5guty49ArDTwzJ/yNuPIbZppyRxFQsRCWrocxIY+CnMVG+qfbU2FmDKyvSGClow==", + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.14.tgz", + "integrity": "sha512-VnXFiIW8yNn9kIHN88xvZ4yOWchftKDsRJ8fEPacX/wl1lOvBrhsJ/OeJCXq7B0AaijRuqgzSKalJoPk+D8MPg==", "dev": true, "dependencies": { - "available-typed-arrays": "^1.0.5", - "call-bind": "^1.0.4", + "available-typed-arrays": "^1.0.6", + "call-bind": "^1.0.5", "for-each": "^0.3.3", "gopd": "^1.0.1", - "has-tostringtag": "^1.0.0" + "has-tostringtag": "^1.0.1" }, "engines": { "node": ">= 0.4" From b138f67e86a8af193762395bd985c5c9ec8cc00f Mon Sep 17 00:00:00 2001 From: Jackson Weber Date: Fri, 2 Feb 2024 14:24:10 -0800 Subject: [PATCH 9/9] Update EndToEnd.tests.ts --- Tests/EndToEnd.tests.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/EndToEnd.tests.ts b/Tests/EndToEnd.tests.ts index a51b6571..c6149754 100644 --- a/Tests/EndToEnd.tests.ts +++ b/Tests/EndToEnd.tests.ts @@ -484,7 +484,7 @@ describe("EndToEnd", () => { writeFileSync = sandbox.stub(fs, 'writeFileSync'); existsSync = sandbox.stub(fs, 'existsSync').returns(true); readdir = sandbox.stub(FileSystemHelper, 'readdirAsync').returns(['1.ai.json']); - readdirSync = sandbox.stub(fs, 'readdirSync').returns(['.ai.json']); + readdirSync = sandbox.stub(fs, 'readdirSync').returns(['1.ai.json']); stat = sandbox.stub(FileSystemHelper, 'statAsync').returns({ isFile: () => true, size: 8000 }); statSync = sandbox.stub(fs, 'statSync').returns({ isFile: () => true, size: 8000 }); lstat = sandbox.stub(FileSystemHelper, 'lstatAsync').returns({ isDirectory: () => true });