feat: Add script to add users for bulk import

bulkimport_scripts/add_users_for_bulk_import/.gitignore

@@ -0,0 +1,3 @@
+node_modules
+./usersHavingInvalidSchema.json
+./remainingUsers.json

bulkimport_scripts/add_users_for_bulk_import/README.md

@@ -0,0 +1,29 @@
+# Add Users For Bulk Import Script
+
+The `/bulk-import/users` POST API endpoint in SuperTokens Core allows to add users to the database to bulk import the users. However, the API only allows importing 10,000 users at once. This script can take a JSON file containing a large number of users and call the API in batches of 10,000.
+
+## How to Run
+
+1. Ensure you have Node.js (V16 or higher) installed on your system.
+2. Open a terminal window and navigate to the directory where the script is located.
+3. Run the following command:
+
+   ```
+   node index.js <CoreAPIURL> <InputFileName>
+   ```
+
+   Replace `<CoreAPIURL>` with the URL of the core API endpoint and `<InputFileName>` with the path to the input JSON file containing user data.
+
+## Format of Input File
+
+The input file should be a JSON file with the same format as requested by the `/bulk-import/users` POST API endpoint.
+
+## Expected Outputs
+
+- Upon successful execution, the script will output a summary message indicating the number of users processed, any remaining users, and any users with invalid schema.
+- If there are remaining users to be processed, a file named `remainingUsers.json` will be generated, containing the details of remaining users.
+- If there are users with invalid schema, a file named `usersHavingInvalidSchema.json` will be generated, containing the details of users with invalid schema.
+
+## Note
+
+The script would re-write the `remainingUsers.json` and `usersHavingInvalidSchema.json` files on each run. Ensure to back up these files if needed.

bulkimport_scripts/add_users_for_bulk_import/index.js

@@ -0,0 +1,167 @@
+/*
+ *    Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved.
+ *
+ *    This software is licensed under the Apache License, Version 2.0 (the
+ *    "License") as published by the Apache Software Foundation.
+ *
+ *    You may not use this file except in compliance with the License. You may
+ *    obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    License for the specific language governing permissions and limitations
+ *    under the License.
+ */
+
+const fs = require('fs/promises');
+const process = require('process');
+
+const BATCH_SIZE = 10000;
+const USERS_HAVING_INVALID_SCHEMA_FILE = './usersHavingInvalidSchema.json';
+const REMAINING_USERS_FILE = './remainingUsers.json';
+
+function parseInputArgs() {
+  if (process.argv.length !== 4) {
+    console.error('Usage: node index.js <CoreAPIURL> <InputFileName>');
+    process.exit(1);
+  }
+
+  return { coreAPIUrl: process.argv[2], inputFileName: process.argv[3] };
+}
+
+
+async function getUsersFromInputFile({ inputFileName }) {
+  try {
+    const inputFileDataString = await fs.readFile(inputFileName, 'utf8');
+    const inputFileData = JSON.parse(inputFileDataString);
+
+    if (!inputFileData.users || !Array.isArray(inputFileData.users) || inputFileData.users.length === 0) {
+      throw new Error('Expected users array in the input file.');
+    }
+
+    return inputFileData.users;
+  } catch (error) {
+    console.error('Error reading or parsing input file:', error.message);
+    process.exit(1);
+  }
+}
+
+async function deleteFailedToAddUsersFileIfExists() {
+  try {
+    await fs.rm(USERS_HAVING_INVALID_SCHEMA_FILE);
+  } catch (error) {
+    if (error.code !== 'ENOENT') {
+      console.error(`Failed to delete ${USERS_HAVING_INVALID_SCHEMA_FILE}:`, error.message);
+    }
+  }
+}
+
+async function addInvalidSchemaUsersToFile({ errors, users }) {
+  let parsedData = null;
+  try {
+    const existingData = await fs.readFile(USERS_HAVING_INVALID_SCHEMA_FILE, 'utf8');
+    parsedData = JSON.parse(existingData);
+  } catch (error) {
+    if (error.code === 'ENOENT') {
+      parsedData = { users: [] };
+    } else {
+      console.error(`Failed to read output file. Error: ${error.message}`);
+      throw error;
+    }
+  }
+
+  parsedData.users.push(...errors.map((err, index) => ({ user: users[index], errors: err.errors })));
+
+  await fs.writeFile(USERS_HAVING_INVALID_SCHEMA_FILE, JSON.stringify(parsedData, null, 2));
+
+  return users.filter((_, index) => !errors.some(err => err.index === index));
+}
+
+async function updateRemainingUsersFile({ users, index }) {
+  const remainingUsers = users.slice(index + 1);
+  await fs.writeFile(REMAINING_USERS_FILE, JSON.stringify({ users: remainingUsers }, null, 2));
+}
+
+async function removeRemainingUsersFile() {
+  try {
+    await fs.rm(REMAINING_USERS_FILE);
+  } catch (error) {
+    if (error.code !== 'ENOENT') {
+      console.error(`Failed to delete ${REMAINING_USERS_FILE}:`, error.message);
+    }
+  }
+}
+
+async function main() {
+  const { coreAPIUrl, inputFileName } = parseInputArgs();
+
+  const users = await getUsersFromInputFile({ inputFileName });
+
+  await deleteFailedToAddUsersFileIfExists();
+  await updateRemainingUsersFile({ users, index: 0 });
+
+  let usersToProcessInBatch = [];
+  let usersHavingInvalidSchemaCount = 0;
+  let i = 0;
+
+  try {
+    while (i < users.length || usersToProcessInBatch.length > 0) {
+      let remainingBatchSize = usersToProcessInBatch.length > BATCH_SIZE ? 0 : BATCH_SIZE - usersToProcessInBatch.length;
+      remainingBatchSize = Math.min(remainingBatchSize, users.length - i);
+
+      usersToProcessInBatch.push(...users.slice(i, i + remainingBatchSize));
+
+      const res = await fetch(`${coreAPIUrl}/bulk-import/users`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({ users: usersToProcessInBatch }),
+      });
+
+      if (!res.ok && res.status !== 400) {
+        const text = await res.text();
+        console.error(`Failed to add users. API response - status: ${res.status} body: ${text}`);
+        break;
+      }
+
+      if (res.status === 400) {
+        const errors = await res.json();
+        usersHavingInvalidSchemaCount += errors.users.length;
+        usersToProcessInBatch = await addInvalidSchemaUsersToFile({ errors: errors.users, users: usersToProcessInBatch });
+      } else {
+        await updateRemainingUsersFile({ users, index: i });
+        usersToProcessInBatch = [];
+      }
+
+      i += remainingBatchSize;
+    }
+  } catch (error) {
+    console.log(`Got an unexpected Error: `, error);
+  }
+
+  const result = {
+    totalUsers: users.length,
+    processedUsers: i,
+    remainingUsers: users.length - i,
+    usersHavingInvalidSchema: usersHavingInvalidSchemaCount,
+    ...(users.length - i > 0 && { remainingUsersFileName: REMAINING_USERS_FILE }),
+    ...(usersHavingInvalidSchemaCount > 0 && { usersHavingInvalidSchemaFileName: USERS_HAVING_INVALID_SCHEMA_FILE }),
+  };
+
+  if (i < users.length) {
+    const message = usersHavingInvalidSchemaCount > 0 ?
+      `We processed ${i} users and ${usersHavingInvalidSchemaCount} users have invalid schema! Remaining users can be processed again by processing the ${REMAINING_USERS_FILE} file and users having invalid schema needs to be fixed and processed again by processing the ${USERS_HAVING_INVALID_SCHEMA_FILE} file.`
+      : `We processed ${i} users and ${users.length - i} users are remaining to be processed! Remaining users can be processed again by processing the ${REMAINING_USERS_FILE} file.`;
+    console.log({ message, ...result });
+  } else {
+    await removeRemainingUsersFile();
+    const message = usersHavingInvalidSchemaCount > 0 ?
+      `All users processed but ${usersHavingInvalidSchemaCount} users have invalid schema! Users having invalid schema needs to be fixed and processed again by processing the ${USERS_HAVING_INVALID_SCHEMA_FILE} file.` : `All users processed successfully!`;
+    console.log({ message, ...result }); ``
+  }
+}
+
+
+main()

bulkimport_scripts/add_users_for_bulk_import/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "add_users_for_bulk_import",
+  "version": "1.0.0",
+  "engines": {
+    "node": ">=16.0.0"
+  },
+  "description": "A script that takes a JSON file containing user data and calls the bulk import API to add users to the database for processing.",
+  "main": "index.js",
+  "scripts": {
+    "start": "node index.js"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "Apache-2.0",
+  "dependencies": {}
+}