updates

transcend-io · Dec 3, 2024 · e2c9e35 · e2c9e35
1 parent f2935b2
commit e2c9e35
Show file tree

Hide file tree

Showing 7 changed files with 40 additions and 22 deletions.
diff --git a/src/preference-management/getPreferencesForIdentifiers.ts b/src/preference-management/getPreferencesForIdentifiers.ts
@@ -87,13 +87,19 @@ export async function getPreferencesForIdentifiers(
           // continue
         }
         const msg = err?.response?.body || err?.message || '';
-        if (!msg.includes('ETIMEDOUT')) {
+        if (!msg.includes('ETIMEDOUT' || '504 Gateway Time-out')) {
           throw new Error(
             `Received an error from server: ${
               err?.response?.body || err?.message
             }`,
           );
         }
+        logger.warn(
+          colors.yellow(
+            '[RETRYING FAILED REQUEST] ' +
+              `Failed to fetch ${group.length} user preferences from partition ${partitionKey}: ${msg}`,
+          ),
+        );
         const rawResult = await sombra
           .post(`v1/preferences/${partitionKey}/query`, {
             json: {
@@ -113,7 +119,7 @@ export async function getPreferencesForIdentifiers(
       }
     },
     {
-      concurrency: 10,
+      concurrency: 100,
     },
   );
 

diff --git a/src/requests/bulkRestartRequests.ts b/src/requests/bulkRestartRequests.ts
@@ -201,7 +201,7 @@ export async function bulkRestartRequests({
           coreIdentifier: requestResponse.coreIdentifier,
           attemptedAt: new Date().toISOString(),
         });
-        state.setValue(restartedRequests, 'restartedRequests');
+        await state.setValue(restartedRequests, 'restartedRequests');
       } catch (err) {
         const msg = `${err.message} - ${JSON.stringify(
           err.response?.body,
@@ -219,7 +219,7 @@ export async function bulkRestartRequests({
           attemptedAt: new Date().toISOString(),
           error: clientError || msg,
         });
-        state.setValue(failingRequests, 'failingRequests');
+        await state.setValue(failingRequests, 'failingRequests');
       }
       total += 1;
       progressBar.update(total);

diff --git a/src/requests/mapColumnsToAttributes.ts b/src/requests/mapColumnsToAttributes.ts
@@ -53,9 +53,11 @@ export async function mapColumnsToAttributes(
             };
           }),
         );
-  Object.entries(attributeNameMap).forEach(([k, v]) => {
-    state.setValue(v, 'attributeNames', k);
-  });
+  await Promise.all(
+    Object.entries(attributeNameMap).map(([k, v]) =>
+      state.setValue(v, 'attributeNames', k),
+    ),
+  );
 
   return {
     ...state.getValue('attributeNames'),

diff --git a/src/requests/mapColumnsToIdentifiers.ts b/src/requests/mapColumnsToIdentifiers.ts
@@ -59,9 +59,11 @@ export async function mapColumnsToIdentifiers(
             };
           }),
         );
-  Object.entries(identifierNameMap).forEach(([k, v]) => {
-    state.setValue(v, 'identifierNames', k);
-  });
+  await Promise.all(
+    Object.entries(identifierNameMap).map(([k, v]) =>
+      state.setValue(v, 'identifierNames', k),
+    ),
+  );
 
   return {
     ...state.getValue('identifierNames'),

diff --git a/src/requests/mapCsvColumnsToApi.ts b/src/requests/mapCsvColumnsToApi.ts
@@ -59,8 +59,10 @@ export async function mapCsvColumnsToApi(
           }),
         );
 
-  getEntries(columnNameMap).forEach(([k, v]) => {
-    state.setValue(v, 'columnNames', k);
-  });
+  await Promise.all(
+    getEntries(columnNameMap).map(([k, v]) =>
+      state.setValue(v, 'columnNames', k),
+    ),
+  );
   return columnNameMap;
 }
diff --git a/src/requests/mapRequestEnumValues.ts b/src/requests/mapRequestEnumValues.ts
@@ -56,7 +56,10 @@ export async function mapRequestEnumValues(
       Object.values(RequestAction),
       state.getValue('requestTypeToRequestAction'),
     );
-  state.setValue(requestTypeToRequestAction, 'requestTypeToRequestAction');
+  await state.setValue(
+    requestTypeToRequestAction,
+    'requestTypeToRequestAction',
+  );
 
   // Map data subject type
   logger.info(colors.magenta('Determining mapping of columns for subject'));
@@ -66,7 +69,7 @@ export async function mapRequestEnumValues(
       internalSubjects.map(({ type }) => type),
       state.getValue('subjectTypeToSubjectName'),
     );
-  state.setValue(subjectTypeToSubjectName, 'subjectTypeToSubjectName');
+  await state.setValue(subjectTypeToSubjectName, 'subjectTypeToSubjectName');
 
   // Map locale
   logger.info(colors.magenta('Determining mapping of columns for locale'));
@@ -75,7 +78,7 @@ export async function mapRequestEnumValues(
     Object.values(LanguageKey),
     state.getValue('languageToLocale'),
   );
-  state.setValue(languageToLocale, 'languageToLocale');
+  await state.setValue(languageToLocale, 'languageToLocale');
   logger.info(
     colors.magenta('Determining mapping of columns for request status'),
   );
@@ -95,7 +98,7 @@ export async function mapRequestEnumValues(
           [...Object.values(CompletedRequestStatus), NONE],
           state.getValue('statusToRequestStatus'),
         );
-  state.setValue(statusToRequestStatus, 'statusToRequestStatus');
+  await state.setValue(statusToRequestStatus, 'statusToRequestStatus');
 
   // Map country
   logger.info(colors.magenta('Determining mapping of columns for country'));
@@ -110,7 +113,7 @@ export async function mapRequestEnumValues(
           [...Object.values(IsoCountryCode), NONE],
           state.getValue('regionToCountry'),
         );
-  state.setValue(regionToCountry, 'regionToCountry');
+  await state.setValue(regionToCountry, 'regionToCountry');
 
   // Map country sub division
   logger.info(
@@ -127,5 +130,8 @@ export async function mapRequestEnumValues(
           [...Object.values(IsoCountrySubdivisionCode), NONE],
           state.getValue('regionToCountrySubDivision'),
         );
-  state.setValue(regionToCountrySubDivision, 'regionToCountrySubDivision');
+  await state.setValue(
+    regionToCountrySubDivision,
+    'regionToCountrySubDivision',
+  );
 }
diff --git a/src/requests/uploadPrivacyRequestsFromCsv.ts b/src/requests/uploadPrivacyRequestsFromCsv.ts
@@ -259,7 +259,7 @@ export async function uploadPrivacyRequestsFromCsv({
           coreIdentifier: requestResponse.coreIdentifier,
           attemptedAt: new Date().toISOString(),
         });
-        requestState.setValue(successfulRequests, 'successfulRequests');
+        await requestState.setValue(successfulRequests, 'successfulRequests');
       } catch (err) {
         const msg = `${err.message} - ${JSON.stringify(
           err.response?.body,
@@ -286,7 +286,7 @@ export async function uploadPrivacyRequestsFromCsv({
             rowIndex: ind,
             attemptedAt: new Date().toISOString(),
           });
-          requestState.setValue(duplicateRequests, 'duplicateRequests');
+          await requestState.setValue(duplicateRequests, 'duplicateRequests');
         } else {
           const failingRequests = requestState.getValue('failingRequests');
           failingRequests.push({
@@ -295,7 +295,7 @@ export async function uploadPrivacyRequestsFromCsv({
             error: clientError || msg,
             attemptedAt: new Date().toISOString(),
           });
-          requestState.setValue(failingRequests, 'failingRequests');
+          await requestState.setValue(failingRequests, 'failingRequests');
           if (debug) {
             logger.error(colors.red(clientError || msg));
             logger.error(