Merge pull request #132 from skyflowapi/SK-1090-remove-unnecessary-ge…

…t-calls-in-insert-batch-operation-node-sdk

SK-1090 remove unnecessary get calls in insert batch operation

src/vault-api/Controller.ts

@@ -90,10 +90,17 @@ class Controller {
         if (options && options.tokens && typeof options.tokens !== 'boolean') {
           throw new SkyflowError(SKYFLOW_ERROR_CODE.INVALID_TOKENS_IN_INSERT, [], true);
         }
+        if (options && options.continueOnError && typeof options.continueOnError !== 'boolean') {
+          throw new SkyflowError(SKYFLOW_ERROR_CODE.INVALID_CONTINUE_ON_ERROR_IN_INSERT, [], true);
+        }
         if (options) {
-          options = { ...options, tokens: options?.tokens !== undefined ? options.tokens : true };
+          options = { 
+            ...options, 
+            tokens: options?.tokens !== undefined ? options.tokens : true,
+            continueOnError: options?.continueOnError !== undefined ? options.continueOnError : false
+          };
         } else {
-          options = { tokens: true,};
+          options = { tokens: true, continueOnError: false };
         }
         if (options?.upsert) {
           validateUpsertOptions(options.upsert);
@@ -280,7 +287,7 @@ class Controller {
       this.getToken().then((res)=>{
         this.#client
           .request({
-            body: { records: requestBody },
+            body: { ...requestBody },
             requestMethod: 'POST',
             url:
               `${this.#client.config.vaultURL
@@ -294,7 +301,7 @@ class Controller {
             rootResolve(
               constructInsertRecordResponse(
                 response,
-                options.tokens,
+                options,
                 records.records,
               ),
             );

src/vault-api/client/index.ts

@@ -63,6 +63,7 @@ class Client {
       headers: this.getHeaders(data,headerKeys)
       }
     ).then((res)=> {
+      res.data['requestId'] = res.headers['x-request-id']
       resolve(res.data)
     }).catch((err)=> {
         this.failureResponse(err).catch((err)=>reject(err))

src/vault-api/core/Collect.ts

@@ -2,7 +2,7 @@
 	Copyright (c) 2022 Skyflow, Inc. 
 */
 import _ from 'lodash';
-import { IInsertRecordInput, IInsertRecord } from '../utils/common';
+import { IInsertRecordInput, IInsertRecord, IInsertOptions } from '../utils/common';
 
 const getUpsertColumn = (tableName: string, options: Record<string, any>) => {
   let uniqueColumn = '';
@@ -13,13 +13,11 @@ const getUpsertColumn = (tableName: string, options: Record<string, any>) => {
   });
   return uniqueColumn;
 };
-
 export const constructInsertRecordRequest = (
   records: IInsertRecordInput,
   options: Record<string, any> = { tokens: true },
 ) => {
-  const requestBody: any = [];
-  if (options.tokens) {
+  let requestBody: any = [];
     records.records.forEach((record, index) => {
       const upsertColumn = getUpsertColumn(record.table, options);
       requestBody.push({
@@ -28,51 +26,72 @@ export const constructInsertRecordRequest = (
         tableName: record.table,
         fields: record.fields,
         ...(options?.upsert ? { upsert: upsertColumn } : {}),
-      });
-      requestBody.push({
-        method: 'GET',
-        tableName: record.table,
-        ID: `$responses.${2 * index}.records.0.skyflow_id`,
-        tokenization: true,
-      });
-    });
-  } else {
-    records.records.forEach((record) => {
-      const upsertColumn = getUpsertColumn(record.table, options);
-      requestBody.push({
-        method: 'POST',
-        quorum: true,
-        tableName: record.table,
-        fields: record.fields,
-        ...(options?.upsert ? { upsert: upsertColumn } : {}),
+        ...(options?.tokens ? { tokenization: true } : {}),
       });
     });
-  }
+  requestBody = { records: requestBody, continueOnError: options.continueOnError }
   return requestBody;
 };
 
 export const constructInsertRecordResponse = (
   responseBody: any,
-  tokens: boolean,
+  options: IInsertOptions,
   records: IInsertRecord[],
 ) => {
-  if (tokens) {
+  if (options.continueOnError) {
+    const successObjects: any = [];
+    const failureObjects: any= [];
+    responseBody.responses
+    .forEach((response, index) => {
+      const status = response['Status']
+      const body = response['Body']
+      if ('records' in body) {
+        const record = body['records'][0]
+        if (options.tokens) {
+          successObjects.push({
+            table: records[index].table,
+            fields: {
+              skyflow_id: record.skyflow_id,
+              ...record.tokens,
+            },
+            request_index: index,
+          })
+        } else {
+          successObjects.push({
+            table: records[index].table,
+            skyflow_id: record.skyflow_id,
+            request_index: index,
+          })
+        }
+      } else {
+        failureObjects.push({
+          code: status,
+          ddescription: `${body['error']} - requestId: ${responseBody.requestId}`,
+          request_index: index,
+        })
+      }
+    })
+    const finalResponse = {};
+    if (successObjects.length > 0) {
+      finalResponse['records'] = successObjects;
+    }
+    if (failureObjects.length > 0) {
+      finalResponse['errors'] = failureObjects;
+    }
+    return finalResponse;
+  } else if (options.tokens) {
     return {
       records: responseBody.responses
         .map((res, index) => {
-          if (index % 2 !== 0) {
-            const skyflowId = responseBody.responses[index - 1].records[0].skyflow_id;
-            delete res.fields['*'];
+            const skyflowId = responseBody.responses[index].records[0].skyflow_id;
             return {
-              table: records[Math.floor(index/2)].table,
+              table: records[index].table,
               fields: {
                 skyflow_id: skyflowId,
-                ...res.fields,
+                ...res.tokens,
               },
             };
-          }
-          return res;
-        }).filter((res, index) => index % 2 !== 0),
+        }),
     };
   }
   return {

src/vault-api/utils/common/index.ts

@@ -192,10 +192,12 @@ export interface IUpsertOption {
  * Parameters by insert options.
  * @property tokens If `true`, returns tokens for the collected data. Defaults to `false`.
  * @property upsert If specified, upserts data. If not specified, inserts data.
+ * @property continueOnError If specified, decides whether to continue after experiencing failure.
  */
 export interface IInsertOptions {
   tokens?: boolean;
   upsert?: IUpsertOption[];
+  continueOnError?: boolean;
 }
 
 /**

src/vault-api/utils/constants.ts

@@ -87,6 +87,10 @@ const SKYFLOW_ERROR_CODE = {
     code: 400,
     description: logs.errorLogs.INVALID_TOKENS_IN_INSERT,
   },
+  INVALID_CONTINUE_ON_ERROR_IN_INSERT: {
+    code: 400,
+    description: logs.errorLogs.INVALID_TOKENS_IN_INSERT,
+  },
   INVALID_TOKENS_IN_UPDATE: {
     code: 400,
     description: logs.errorLogs.INVALID_TOKENS_IN_UPDATE,

src/vault-api/utils/logs.ts

@@ -119,6 +119,7 @@ const logs = {
     INVALID_TABLE_IN_UPSERT_OPTION: 'Interface: insert method - Invalid table in upsert object at index %s1, table of type non empty string is required.',
     INVALID_COLUMN_IN_UPSERT_OPTION: 'Interface: insert method - Invalid column in upsert object at index %s1, column of type non empty string is required.',
     INVALID_TOKENS_IN_INSERT: 'Interface: insert method - Invalid tokens in options. tokens of type boolean is required.',
+    INVALID_CONTINUE_ON_ERROR_IN_INSERT: 'Interface: insert method - Invalid continueOnError in options. Value of type boolean is required.',
     INVALID_TOKENS_IN_UPDATE: 'Interface: update method - Invalid tokens in options. tokens of type boolean is required.',
     MISSING_TABLE_IN_IN_UPDATE: 'Interface: update method - table key is required in records object at index %s1',
     MISSING_FIELDS_IN_IN_UPDATE: 'Interface: update method - fields key is required in records object at index %s1',

test/vault-api/Client.test.js

@@ -33,7 +33,7 @@ describe("Client Class",()=>{
       const data = JSON.stringify({ name: "John Doe", age: 30 });
       const headers = { "content-type": "application/json","sky-metadata":JSON.stringify(generateSDKMetrics()) };
       axios.mockImplementation(() =>
-        Promise.resolve({ data: { message: "Success" } })
+        Promise.resolve({ data: { message: "Success" }, headers: { "x-request-id": "22r5-dfbf-3543" }})
       );
 
       const response = await client.request(request);
@@ -44,7 +44,7 @@ describe("Client Class",()=>{
         data: data,
         headers: headers,
       });
-      expect(response).toEqual({ message: "Success" });
+      expect(response).toEqual({ message: "Success", requestId: "22r5-dfbf-3543" });
     });
 
     test("should return an error if the request to client fails", async () => {

test/vault-api/Skyflow.test.js

@@ -131,8 +131,10 @@ const options = {
   tokens: true,
 };
 
-const insertResponse = {"vaultID":"<VaultID>","responses":[{"records":[{"skyflow_id":"id"}]},{"fields":{"card_number":"token","cvv":"token","expiry_date":"token","fullname":"token"}}]}
+const insertResponse = {"vaultID":"<VaultID>","responses":[{"records":[{"skyflow_id":"id","tokens":{"card_number":"token","cvv":"token","expiry_date":"token","fullname":"token"}}]}]}
 const insertResponseWithoutTokens = {"vaultID":"<VaultID>","responses":[{"records":[{"skyflow_id":"id"}]}]}
+const insertResponseCOEWithTokens = {"vaultID":"<VaultID>","responses":[{"Status":400,"Body":{"error":"Error Inserting Records due to unique constraint violation"}},{"Status":200,"Body":{"records":[{"skyflow_id":"id","tokens":{"card_number":"token","cvv":"token","expiry_date":"token","fullname":"token"}}]}}]}
+const insertResponseCOEWithoutTokens = {"vaultID":"<VaultID>","responses":[{"Status":400,"Body":{"error":"Error Inserting Records due to unique constraint violation"}},{"Status":200,"Body":{"records":[{"skyflow_id":"id"}]}}]}
 const on = jest.fn();
 
 describe('skyflow insert', () => {
@@ -144,12 +146,10 @@ describe('skyflow insert', () => {
       vaultURL: 'https://www.vaulturl.com',
       getBearerToken: ()=>{
         return new Promise((resolve,_)=>{
-            resolve("token")
+          resolve("token")
         })
       }
     });
-
-
   });
 
   test('insert invalid input', (done) => {
@@ -199,7 +199,6 @@ describe('skyflow insert', () => {
 
   });
 
-
   test('insert success without tokens', () => {
 
 
@@ -317,6 +316,7 @@ describe('skyflow insert', () => {
           done(err);
         }
   });
+
   test('insert without any options',(done)=>{
     try{
       skyflow = Skyflow.init({
@@ -338,6 +338,91 @@ describe('skyflow insert', () => {
     }
   });
 
+  test('insert with invalid continueOnError option type', (done) => {
+    try {
+      const res = skyflow.insert(records, { continueOnError: {} });
+      res.catch((err) => {
+        expect(err).toBeDefined();
+        done();
+      });
+    } catch (err) {
+      done(err);
+    }
+  });
+
+  test('insert success with continueOnError as true with tokens', (done) => {
+    try {
+      jest.mock('../../src/vault-api/utils/jwt-utils', () => ({
+        __esModule: true,
+        isTokenValid:jest.fn(() => true),
+      }));
+      const clientReq = jest.fn(() => Promise.resolve(insertResponseCOEWithTokens));
+      const mockClient = {
+        config: skyflowConfig,
+        request: clientReq,
+        metadata: {}
+      }
+      setLogLevel(LogLevel.WARN)
+      clientModule.mockImplementation(() => { return mockClient });
+
+      skyflow = Skyflow.init({
+        vaultID: '<VaultID>',
+        vaultURL: 'https://www.vaulturl.com',
+        getBearerToken: () => {
+          return new Promise((resolve, _) => {
+            resolve("token");
+          })
+        }
+      });
+      const res = skyflow.insert({
+        records: [
+          records['records'][0],
+          records['records'][0]
+        ]
+      }, { tokens: true, continueOnError: true });
+      res.then((res) => {
+        expect(clientReq).toHaveBeenCalled();
+        expect(res.records.length).toBe(1);
+        expect(res.errors.length).toBe(1); 
+        done();
+      });
+    } catch (err) {
+      done(err);
+    }
+  });
+
+  test('insert success with continueOnError as true without tokens', (done) => {
+    try {
+      jest.mock('../../src/vault-api/utils/jwt-utils', () => ({
+        __esModule: true,
+        isTokenValid:jest.fn(() => true),
+      }));
+      const clientReq = jest.fn(() => Promise.resolve(insertResponseCOEWithoutTokens));
+      const mockClient = {
+        config: skyflowConfig,
+        request: clientReq,
+        metadata: {}
+      }
+      setLogLevel(LogLevel.WARN)
+      clientModule.mockImplementation(() => { return mockClient });
+
+      skyflow = Skyflow.init(skyflowConfig);
+      const res = skyflow.insert({
+        records: [
+          records['records'][0],
+          records['records'][0]
+        ]
+      }, { tokens: false, continueOnError: true });
+      res.then((res) => {
+        expect(clientReq).toHaveBeenCalled();
+        expect(res.records.length).toBe(1);
+        expect(res.errors.length).toBe(1); 
+        done();
+      });
+    } catch (err) {
+      done(err);
+    }
+  });
 });
 
 const detokenizeInput = {