feat: multi-stage-output/new csv flags (#1110)

* fix: cache refactor, remove unused code/props

* chore: big refactor

* test: nuke UTs

* fix: keep cache entry

* chore: more refactoring

* fix: bring back verbose table

* test: update HRO assertion

* fix: bulk delete can return JSON failures

* fix: match regex for table

* fix: bulk upsert return json failures + NUT

* chore: remove unused code/refactor

* fix: handle hardDelete missing perm error

* chore: address final TODOs

* chore: code review

* chore: update msg

* chore: remove column-delimiter for bulk delete

* test: nuts use new flags

* fix: handle quoted fields when detecting column delimiter

* chore: removed dead code

* chore: refactor

* fix: remove ref to deleted md

* test: add import -> export -> import nut

* fix: throw if job didn't process any record

* fix: pass err action tokens

* fix: restore support for resuming non-local jobs

* fix: dont get records if job is in progress

* fix: bump jsforce-node

command-snapshot.json

@@ -45,6 +45,7 @@
       "flags-dir",
       "hard-delete",
       "json",
+      "line-ending",
       "loglevel",
       "sobject",
       "target-org",
@@ -309,10 +310,12 @@
     "flags": [
       "api-version",
       "async",
+      "column-delimiter",
       "external-id",
       "file",
       "flags-dir",
       "json",
+      "line-ending",
       "loglevel",
       "sobject",
       "target-org",

messages/bulkIngest.md

@@ -37,6 +37,59 @@ Job has been aborted.
 - Get the job results by running: "sf data bulk results -o %s --job-id %s".
 - View the job in the org: "sf org open -o %s --path '/lightning/setup/AsyncApiJobStatus/page?address=%2F%s'".
 
+# error.hardDeletePermission
+
+You must have the "Bulk API Hard Delete" system permission to use the --hard-delete flag. This permission is disabled by default and can be enabled only by a system administrator.
+
+# error.noProcessedRecords
+
+Job finished successfully but it didn't process any record.
+
+# error.noProcessedRecords.actions
+
+- Check that the provided CSV file is valid.
+- View the job in the org: "sf org open -o %s --path '/lightning/setup/AsyncApiJobStatus/page?address=%2F%s'".
+
 # flags.column-delimiter.summary
 
-Column delimiter used in the CSV file. Default is COMMA.
+Column delimiter used in the CSV file.
+
+# flags.line-ending.summary
+
+Line ending used in the CSV file. Default value on Windows is `CRLF`; on macOS and Linux it's `LF`.
+
+# flags.sobject.summary
+
+API name of the Salesforce object, either standard or custom, that you want to update or delete records from.
+
+# flags.csvfile.summary
+
+CSV file that contains the IDs of the records to update or delete.
+
+# flags.wait.summary
+
+Number of minutes to wait for the command to complete before displaying the results.
+
+# flags.async.summary
+
+Run the command asynchronously.
+
+# flags.verbose.summary
+
+Print verbose output of failed records if result is available.
+
+# flags.jobid
+
+ID of the job you want to resume.
+
+# flags.useMostRecent.summary
+
+Use the ID of the most recently-run bulk job.
+
+# flags.targetOrg.summary
+
+Username or alias of the target org. Not required if the "target-org" configuration variable is already set.
+
+# flags.wait.summary
+
+Number of minutes to wait for the command to complete before displaying the results.

messages/data.import.bulk.md

@@ -41,7 +41,3 @@ Time to wait for the command to finish, in minutes.
 # flags.line-ending.summary
 
 Line ending used in the CSV file. Default value on Windows is `CRLF`; on macOS and Linux it's `LF`.
-
-# flags.column-delimiter.summary
-
-Column delimiter used in the CSV file. Default is COMMA.

messages/data.update.bulk.md

@@ -37,11 +37,3 @@ CSV file that contains the Salesforce object records you want to update.
 # flags.sobject.summary
 
 API name of the Salesforce object, either standard or custom, which you are updating.
-
-# flags.line-ending.summary
-
-Line ending used in the CSV file. Default value on Windows is `CRLF`; on macOS and Linux it's `LF`.
-
-# flags.column-delimiter.summary
-
-Column delimiter used in the CSV file. Default is COMMA.

messages/messages.md

@@ -1,28 +1,3 @@
-# success
-
-Bulk %s request %s started successfully.
-
-# checkStatus
-
-Run the command "sf data %s resume -i %s -o %s" to check the status.
-
-# checkJobViaUi
-
-To review the details of this job, run:
-sf org open --target-org %s --path "/lightning/setup/AsyncApiJobStatus/page?address=%2F%s"
-
-# remainingTimeStatus
-
-Remaining time: %d minutes.
-
-# remainingRecordsStatus
-
-Processed %d | Success %d | Fail %d
-
-# bulkJobFailed
-
-The bulk job %s failed. Check the job status for more information.
-
 # perfLogLevelOption
 
 Get API performance data.
@@ -48,10 +23,6 @@ Malformed key=value pair for value: %s.
 
 Format to display the results; the --json flag overrides this flag.
 
-# bulkRequestIdRequiredWhenNotUsingMostRecent
-
-The bulk request id must be supplied when not looking for most recent cache entry.
-
 # error.bulkRequestIdNotFound
 
 Could not find a cache entry for job ID %s.
@@ -60,9 +31,9 @@ Could not find a cache entry for job ID %s.
 
 Could not load a most recent cache entry for a bulk request. Please rerun your command with a bulk request id.
 
-# cannotCreateResumeOptionsWithoutAnOrg
+# error.skipCacheValidateNoOrg
 
-Cannot create a cache entry without a valid org.
+A default target org for the job %s is required to be set because the job isn't in the local cache.
 
 # usernameRequired
 

package.json

@@ -118,7 +118,8 @@
     "version": "oclif readme"
   },
   "dependencies": {
-    "@jsforce/jsforce-node": "^3.6.2",
+    "@jsforce/jsforce-node": "^3.6.3",
+    "@oclif/multi-stage-output": "^0.7.5",
     "@oclif/multi-stage-output": "^0.7.12",
     "@salesforce/core": "^8.6.1",
     "@salesforce/kit": "^3.2.2",

src/bulkDataRequestCache.ts

@@ -5,9 +5,9 @@
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
 
-import { TTLConfig, Global, Logger, Messages, Org } from '@salesforce/core';
+import { TTLConfig, Global, Logger, Messages, Org, ConfigAggregator, OrgConfigProperties } from '@salesforce/core';
 import { Duration } from '@salesforce/kit';
-import type { ResumeBulkExportOptions, ResumeBulkImportOptions, ResumeOptions } from './types.js';
+import type { ResumeBulkExportOptions, ResumeBulkImportOptions } from './types.js';
 import { ColumnDelimiterKeys } from './bulkUtils.js';
 
 Messages.importMessagesDirectoryFromMetaUrl(import.meta.url);
@@ -64,61 +64,58 @@ export abstract class BulkDataRequestCache extends TTLConfig<TTLConfig.Options,
     Logger.childFromRoot('DataRequestCache').debug(`bulk cache saved for ${bulkRequestId}`);
   }
 
+  /**
+   * Resolve entries from the local cache.
+   *
+   * @param jobIdOrMostRecent job ID or boolean value to decide if it should return the most recent entry in the cache.
+   * @param skipCacheValidatation make this method not throw if you passed a job ID that's not in the cache
+   * This was only added for `data upsert/delete resume` for backwards compatibility and will be removed after March 2025.
+   */
   public async resolveResumeOptionsFromCache(
-    bulkJobId: string | undefined,
-    useMostRecent: boolean,
-    org: Org | undefined,
-    apiVersion: string | undefined
-  ): Promise<ResumeOptions> {
-    if (!useMostRecent && !bulkJobId) {
-      throw messages.createError('bulkRequestIdRequiredWhenNotUsingMostRecent');
-    }
-    const resumeOptions = {
-      operation: 'query',
-      query: '',
-      pollingOptions: { pollTimeout: 0, pollInterval: 0 },
-    } satisfies Pick<ResumeOptions['options'], 'operation' | 'query' | 'pollingOptions'>;
-
-    if (useMostRecent) {
+    jobIdOrMostRecent: string | boolean,
+    skipCacheValidatation = false
+  ): Promise<ResumeBulkImportOptions> {
+    if (typeof jobIdOrMostRecent === 'boolean') {
       const key = this.getLatestKey();
-      if (key) {
-        // key definitely exists because it came from the cache
-        const entry = this.get(key);
-
-        return {
-          jobInfo: { id: entry.jobId },
-          options: {
-            ...resumeOptions,
-            connection: (await Org.create({ aliasOrUsername: entry.username })).getConnection(apiVersion),
-          },
-        };
-      }
-    }
-    if (bulkJobId) {
-      const entry = this.get(bulkJobId);
-      if (entry) {
-        return {
-          jobInfo: { id: entry.jobId },
-          options: {
-            ...resumeOptions,
-            connection: (await Org.create({ aliasOrUsername: entry.username })).getConnection(apiVersion),
-          },
-        };
-      } else if (org) {
-        return {
-          jobInfo: { id: bulkJobId },
-          options: {
-            ...resumeOptions,
-            connection: org.getConnection(apiVersion),
-          },
-        };
-      } else {
-        throw messages.createError('cannotCreateResumeOptionsWithoutAnOrg');
+      if (!key) {
+        throw messages.createError('error.missingCacheEntryError');
       }
-    } else if (useMostRecent) {
-      throw messages.createError('error.missingCacheEntryError');
+      // key definitely exists because it came from the cache
+      const entry = this.get(key);
+
+      return {
+        jobInfo: { id: entry.jobId },
+        options: {
+          connection: (await Org.create({ aliasOrUsername: entry.username })).getConnection(),
+        },
+      };
     } else {
-      throw messages.createError('bulkRequestIdRequiredWhenNotUsingMostRecent');
+      const entry = this.get(jobIdOrMostRecent);
+      if (!entry) {
+        if (skipCacheValidatation) {
+          const config = await ConfigAggregator.create();
+          const aliasOrUsername = config.getInfo(OrgConfigProperties.TARGET_ORG)?.value as string;
+          if (!aliasOrUsername) {
+            throw messages.createError('error.skipCacheValidateNoOrg', [jobIdOrMostRecent]);
+          }
+
+          return {
+            jobInfo: { id: jobIdOrMostRecent },
+            options: {
+              connection: (await Org.create({ aliasOrUsername })).getConnection(),
+            },
+          };
+        }
+
+        throw messages.createError('error.bulkRequestIdNotFound', [jobIdOrMostRecent]);
+      }
+
+      return {
+        jobInfo: { id: entry.jobId },
+        options: {
+          connection: (await Org.create({ aliasOrUsername: entry.username })).getConnection(),
+        },
+      };
     }
   }
 }
@@ -377,12 +374,6 @@ export class BulkExportRequestCache extends TTLConfig<TTLConfig.Options, BulkExp
     jobIdOrMostRecent: string | boolean,
     apiVersion: string | undefined
   ): Promise<ResumeBulkExportOptions> {
-    const resumeOptionsOptions = {
-      operation: 'query',
-      query: '',
-      pollingOptions: { pollTimeout: 0, pollInterval: 0 },
-    } satisfies Pick<ResumeOptions['options'], 'operation' | 'query' | 'pollingOptions'>;
-
     if (typeof jobIdOrMostRecent === 'boolean') {
       const key = this.getLatestKey();
       if (!key) {
@@ -399,7 +390,6 @@ export class BulkExportRequestCache extends TTLConfig<TTLConfig.Options, BulkExp
           columnDelimiter: entry.outputInfo.columnDelimiter,
         },
         options: {
-          ...resumeOptionsOptions,
           connection: (await Org.create({ aliasOrUsername: entry.username })).getConnection(apiVersion),
         },
       };
@@ -413,7 +403,6 @@ export class BulkExportRequestCache extends TTLConfig<TTLConfig.Options, BulkExp
         jobInfo: { id: entry.jobId },
         outputInfo: entry.outputInfo,
         options: {
-          ...resumeOptionsOptions,
           connection: (await Org.create({ aliasOrUsername: entry.username })).getConnection(apiVersion),
         },
       };