From 451289b2daaf879bfa99a9c6fb3685e1b657d4a3 Mon Sep 17 00:00:00 2001 From: Sattvik Chakravarthy Date: Wed, 13 Mar 2024 13:01:13 +0530 Subject: [PATCH 1/5] fix: pass appId to getUserIdMappingForSuperTokensIds --- .../supertokens/storage/postgresql/Start.java | 4 ++-- .../queries/EmailVerificationQueries.java | 4 ++-- .../queries/UserIdMappingQueries.java | 23 ++++++++++++------- .../postgresql/test/OneMillionUsersTest.java | 3 +++ 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/main/java/io/supertokens/storage/postgresql/Start.java b/src/main/java/io/supertokens/storage/postgresql/Start.java index cf611631..4e2ee20e 100644 --- a/src/main/java/io/supertokens/storage/postgresql/Start.java +++ b/src/main/java/io/supertokens/storage/postgresql/Start.java @@ -2231,10 +2231,10 @@ public boolean updateOrDeleteExternalUserIdInfo(AppIdentifier appIdentifier, Str } @Override - public HashMap getUserIdMappingForSuperTokensIds(ArrayList userIds) + public HashMap getUserIdMappingForSuperTokensIds(AppIdentifier appIdentifier, ArrayList userIds) throws StorageQueryException { try { - return UserIdMappingQueries.getUserIdMappingWithUserIds(this, userIds); + return UserIdMappingQueries.getUserIdMappingWithUserIds(this, appIdentifier, userIds); } catch (SQLException e) { throw new StorageQueryException(e); } diff --git a/src/main/java/io/supertokens/storage/postgresql/queries/EmailVerificationQueries.java b/src/main/java/io/supertokens/storage/postgresql/queries/EmailVerificationQueries.java index ff9fc950..6fd00660 100644 --- a/src/main/java/io/supertokens/storage/postgresql/queries/EmailVerificationQueries.java +++ b/src/main/java/io/supertokens/storage/postgresql/queries/EmailVerificationQueries.java @@ -271,7 +271,7 @@ public static List isEmailVerified_transaction(Start start, Connection s // calculating the verified emails HashMap supertokensUserIdToExternalUserIdMap = UserIdMappingQueries.getUserIdMappingWithUserIds_Transaction(start, - sqlCon, supertokensUserIds); + sqlCon, appIdentifier, supertokensUserIds); HashMap externalUserIdToSupertokensUserIdMap = new HashMap<>(); List supertokensOrExternalUserIdsToQuery = new ArrayList<>(); @@ -340,7 +340,7 @@ public static List isEmailVerified(Start start, AppIdentifier appIdentif // We have external user id stored in the email verification table, so we need to fetch the mapped userids for // calculating the verified emails HashMap supertokensUserIdToExternalUserIdMap = UserIdMappingQueries.getUserIdMappingWithUserIds(start, - supertokensUserIds); + appIdentifier, supertokensUserIds); HashMap externalUserIdToSupertokensUserIdMap = new HashMap<>(); List supertokensOrExternalUserIdsToQuery = new ArrayList<>(); for (String userId : supertokensUserIds) { diff --git a/src/main/java/io/supertokens/storage/postgresql/queries/UserIdMappingQueries.java b/src/main/java/io/supertokens/storage/postgresql/queries/UserIdMappingQueries.java index 24f4fab7..a2388765 100644 --- a/src/main/java/io/supertokens/storage/postgresql/queries/UserIdMappingQueries.java +++ b/src/main/java/io/supertokens/storage/postgresql/queries/UserIdMappingQueries.java @@ -128,7 +128,8 @@ public static UserIdMapping[] getUserIdMappingWithEitherSuperTokensUserIdOrExter } - public static HashMap getUserIdMappingWithUserIds(Start start, List userIds) + public static HashMap getUserIdMappingWithUserIds(Start start, + AppIdentifier appIdentifier, List userIds) throws SQLException, StorageQueryException { if (userIds.size() == 0) { @@ -137,7 +138,8 @@ public static HashMap getUserIdMappingWithUserIds(Start start, L // No need to filter based on tenantId because the id list is already filtered for a tenant StringBuilder QUERY = new StringBuilder( - "SELECT * FROM " + Config.getConfig(start).getUserIdMappingTable() + " WHERE supertokens_user_id IN ("); + "SELECT * FROM " + Config.getConfig(start).getUserIdMappingTable() + " WHERE app_id = ? AND " + + "supertokens_user_id IN ("); for (int i = 0; i < userIds.size(); i++) { QUERY.append("?"); if (i != userIds.size() - 1) { @@ -147,9 +149,10 @@ public static HashMap getUserIdMappingWithUserIds(Start start, L } QUERY.append(")"); return execute(start, QUERY.toString(), pst -> { + pst.setString(1, appIdentifier.getAppId()); for (int i = 0; i < userIds.size(); i++) { - // i+1 cause this starts with 1 and not 0 - pst.setString(i + 1, userIds.get(i)); + // i+2 cause this starts with 1 and not 0, and 1 is appId + pst.setString(i + 2, userIds.get(i)); } }, result -> { HashMap userIdMappings = new HashMap<>(); @@ -161,7 +164,9 @@ public static HashMap getUserIdMappingWithUserIds(Start start, L }); } - public static HashMap getUserIdMappingWithUserIds_Transaction(Start start, Connection sqlCon, List userIds) + public static HashMap getUserIdMappingWithUserIds_Transaction(Start start, Connection sqlCon, + AppIdentifier appIdentifier, + List userIds) throws SQLException, StorageQueryException { if (userIds.size() == 0) { @@ -170,7 +175,8 @@ public static HashMap getUserIdMappingWithUserIds_Transaction(St // No need to filter based on tenantId because the id list is already filtered for a tenant StringBuilder QUERY = new StringBuilder( - "SELECT * FROM " + Config.getConfig(start).getUserIdMappingTable() + " WHERE supertokens_user_id IN ("); + "SELECT * FROM " + Config.getConfig(start).getUserIdMappingTable() + " WHERE app_id = ? AND " + + "supertokens_user_id IN ("); for (int i = 0; i < userIds.size(); i++) { QUERY.append("?"); if (i != userIds.size() - 1) { @@ -180,9 +186,10 @@ public static HashMap getUserIdMappingWithUserIds_Transaction(St } QUERY.append(")"); return execute(sqlCon, QUERY.toString(), pst -> { + pst.setString(1, appIdentifier.getAppId()); for (int i = 0; i < userIds.size(); i++) { - // i+1 cause this starts with 1 and not 0 - pst.setString(i + 1, userIds.get(i)); + // i+2 cause this starts with 1 and not 0, and 1 is appId + pst.setString(i + 2, userIds.get(i)); } }, result -> { HashMap userIdMappings = new HashMap<>(); diff --git a/src/test/java/io/supertokens/storage/postgresql/test/OneMillionUsersTest.java b/src/test/java/io/supertokens/storage/postgresql/test/OneMillionUsersTest.java index 621a2431..5b40b133 100644 --- a/src/test/java/io/supertokens/storage/postgresql/test/OneMillionUsersTest.java +++ b/src/test/java/io/supertokens/storage/postgresql/test/OneMillionUsersTest.java @@ -263,6 +263,7 @@ private void createUserData(Main main) throws Exception { while (true) { UserIdMapping.populateExternalUserIdForUsers( + new AppIdentifier(null, null), (StorageLayer.getBaseStorage(main)), usersResult.users); @@ -389,6 +390,7 @@ private void createSessions(Main main) throws Exception { while (true) { UserIdMapping.populateExternalUserIdForUsers( + new AppIdentifier(null, null), (StorageLayer.getBaseStorage(main)), usersResult.users); @@ -879,6 +881,7 @@ private void measureOperations(Main main) throws Exception { try { UserPaginationContainer users = AuthRecipe.getUsers(main, 1, "ASC", null, null, null); UserIdMapping.populateExternalUserIdForUsers( + new AppIdentifier(null, null), (StorageLayer.getBaseStorage(main)), users.users); From b6f90652d64fe476bc4959437777eba6aa766dfd Mon Sep 17 00:00:00 2001 From: Sattvik Chakravarthy Date: Wed, 13 Mar 2024 17:31:28 +0530 Subject: [PATCH 2/5] fix: one million users test --- .../postgresql/test/OneMillionUsersTest.java | 213 ++++++++---------- 1 file changed, 93 insertions(+), 120 deletions(-) diff --git a/src/test/java/io/supertokens/storage/postgresql/test/OneMillionUsersTest.java b/src/test/java/io/supertokens/storage/postgresql/test/OneMillionUsersTest.java index 5b40b133..1fd1cc1c 100644 --- a/src/test/java/io/supertokens/storage/postgresql/test/OneMillionUsersTest.java +++ b/src/test/java/io/supertokens/storage/postgresql/test/OneMillionUsersTest.java @@ -77,6 +77,12 @@ public void beforeEach() { static int TOTAL_USERS = 1000000; static int NUM_THREADS = 16; + Object lock = new Object(); + Set allUserIds = new HashSet<>(); + Set allPrimaryUserIds = new HashSet<>(); + Map userIdMappings = new HashMap<>(); + Map primaryUserIdMappings = new HashMap<>(); + private void createEmailPasswordUsers(Main main) throws Exception { System.out.println("Creating emailpassword users..."); @@ -103,6 +109,9 @@ private void createEmailPasswordUsers(Main main) throws Exception { storage.signUp(TenantIdentifier.BASE_TENANT, userId, "eptest" + finalI + "@example.com", combinedPasswordHash, timeJoined); + synchronized (lock) { + allUserIds.add(userId); + } } catch (Exception e) { throw new RuntimeException(e); } @@ -129,6 +138,9 @@ private void createPasswordlessUsersWithEmail(Main main) throws Exception { long timeJoined = System.currentTimeMillis(); try { storage.createUser(TenantIdentifier.BASE_TENANT, userId, "pltest" + finalI + "@example.com", null, timeJoined); + synchronized (lock) { + allUserIds.add(userId); + } } catch (Exception e) { throw new RuntimeException(e); } @@ -156,6 +168,9 @@ private void createPasswordlessUsersWithPhone(Main main) throws Exception { long timeJoined = System.currentTimeMillis(); try { storage.createUser(TenantIdentifier.BASE_TENANT, userId, null, "+91987654" + finalI, timeJoined); + synchronized (lock) { + allUserIds.add(userId); + } } catch (Exception e) { throw new RuntimeException(e); } @@ -184,6 +199,9 @@ private void createThirdpartyUsers(Main main) throws Exception { try { storage.signUp(TenantIdentifier.BASE_TENANT, userId, "tptest" + finalI + "@example.com", new LoginMethod.ThirdParty("google", "googleid" + finalI), timeJoined ); + synchronized (lock) { + allUserIds.add(userId); + } } catch (Exception e) { throw new RuntimeException(e); } @@ -211,42 +229,25 @@ private void createUserIdMappings(Main main) throws Exception { System.out.println("Creating user id mappings..."); ExecutorService es = Executors.newFixedThreadPool(NUM_THREADS); - - UserPaginationContainer usersResult = AuthRecipe.getUsers(main, 10000, "ASC", null, - null, null); - AtomicLong usersUpdated = new AtomicLong(0); - while (true) { - for (AuthRecipeUserInfo user : usersResult.users) { - es.execute(() -> { - Random random = new Random(); - - // UserId mapping - for (LoginMethod lm : user.loginMethods) { - String userId = user.getSupertokensUserId(); - - if (random.nextBoolean()) { - userId = "ext" + UUID.randomUUID().toString(); - try { - UserIdMapping.createUserIdMapping(main, lm.getSupertokensUserId(), userId, null, false); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - long count = usersUpdated.incrementAndGet(); - if (count % 10000 == 9999) { - System.out.println("Updated " + (count) + " users"); - } + for (String userId : allUserIds) { + es.execute(() -> { + String extUserId = "ext" + UUID.randomUUID().toString(); + try { + UserIdMapping.createUserIdMapping(main, userId, extUserId, null, false); + synchronized (lock) { + userIdMappings.put(userId, extUserId); } - }); - } - if (usersResult.nextPaginationToken == null) { - break; - } - usersResult = AuthRecipe.getUsers(main, 10000, "ASC", usersResult.nextPaginationToken, - null, null); + } catch (Exception e) { + throw new RuntimeException(e); + } + + long count = usersUpdated.incrementAndGet(); + if (count % 10000 == 9999) { + System.out.println("Updated " + (count) + " users"); + } + }); } es.shutdown(); @@ -258,43 +259,27 @@ private void createUserData(Main main) throws Exception { ExecutorService es = Executors.newFixedThreadPool(NUM_THREADS / 2); - UserPaginationContainer usersResult = AuthRecipe.getUsers(main, 500, "ASC", null, - null, null); - - while (true) { - UserIdMapping.populateExternalUserIdForUsers( - new AppIdentifier(null, null), - (StorageLayer.getBaseStorage(main)), - usersResult.users); - - for (AuthRecipeUserInfo user : usersResult.users) { - es.execute(() -> { - Random random = new Random(); + for (String userId : allPrimaryUserIds) { + es.execute(() -> { + Random random = new Random(); - // User Metadata - JsonObject metadata = new JsonObject(); - metadata.addProperty("random", random.nextDouble()); + // User Metadata + JsonObject metadata = new JsonObject(); + metadata.addProperty("random", random.nextDouble()); - try { - UserMetadata.updateUserMetadata(main, user.getSupertokensOrExternalUserId(), metadata); + try { + UserMetadata.updateUserMetadata(main, userIdMappings.get(userId), metadata); - // User Roles - if (random.nextBoolean()) { - UserRoles.addRoleToUser(main, user.getSupertokensOrExternalUserId(), "admin"); - } else { - UserRoles.addRoleToUser(main, user.getSupertokensOrExternalUserId(), "user"); - } - } catch (Exception e) { - throw new RuntimeException(e); + // User Roles + if (random.nextBoolean()) { + UserRoles.addRoleToUser(main, userIdMappings.get(userId), "admin"); + } else { + UserRoles.addRoleToUser(main, userIdMappings.get(userId), "user"); } - }); - } - - if (usersResult.nextPaginationToken == null) { - break; - } - usersResult = AuthRecipe.getUsers(main, 500, "ASC", usersResult.nextPaginationToken, - null, null); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); } es.shutdown(); @@ -303,25 +288,7 @@ private void createUserData(Main main) throws Exception { private void doAccountLinking(Main main) throws Exception { Set userIds = new HashSet<>(); - - long st = System.currentTimeMillis(); - UserPaginationContainer usersResult = AuthRecipe.getUsers(main, 1000, "ASC", null, - null, null); - - while (true) { - for (AuthRecipeUserInfo user : usersResult.users) { - userIds.add(user.getSupertokensUserId()); - } - if (usersResult.nextPaginationToken == null) { - break; - } - usersResult = AuthRecipe.getUsers(main, 1000, "ASC", usersResult.nextPaginationToken, - null, null); - } - - long en = System.currentTimeMillis(); - - System.out.println("Time taken to get " + TOTAL_USERS + " users (before account linking): " + ((en - st) / 1000) + " sec"); + userIds.addAll(allUserIds); assertEquals(TOTAL_USERS, userIds.size()); @@ -366,6 +333,13 @@ private void doAccountLinking(Main main) throws Exception { throw new RuntimeException(e); } + synchronized (lock) { + allPrimaryUserIds.add(userIdsArray[0]); + for (String userId : userIdsArray) { + primaryUserIdMappings.put(userId, userIdsArray[0]); + } + } + long total = accountsLinked.addAndGet(userIdsArray.length); if (total % 10000 > 9996) { System.out.println("Linked " + (accountsLinked) + " users"); @@ -385,39 +359,24 @@ private void createSessions(Main main) throws Exception { ExecutorService es = Executors.newFixedThreadPool(NUM_THREADS); - UserPaginationContainer usersResult = AuthRecipe.getUsers(main, 500, "ASC", null, - null, null); - - while (true) { - UserIdMapping.populateExternalUserIdForUsers( - new AppIdentifier(null, null), - (StorageLayer.getBaseStorage(main)), - usersResult.users); - - for (AuthRecipeUserInfo user : usersResult.users) { - es.execute(() -> { - try { - for (LoginMethod lM : user.loginMethods) { - String userId = lM.getSupertokensOrExternalUserId(); - SessionInformationHolder session = Session.createNewSession(main, - userId, new JsonObject(), new JsonObject()); - - if (new Random().nextFloat() < 0.05) { - accessToken = session.accessToken.token; - sessionUserId = userId; - } - } + for (String userId : allUserIds) { + String finalUserId = userId; + es.execute(() -> { + try { + SessionInformationHolder session = Session.createNewSession(main, + userIdMappings.get(finalUserId), new JsonObject(), new JsonObject()); - } catch (Exception e) { - throw new RuntimeException(e); + if (new Random().nextFloat() < 0.05) { + synchronized (lock) { + accessToken = session.accessToken.token; + sessionUserId = userIdMappings.get(primaryUserIdMappings.get(finalUserId)); + } } - }); - } - if (usersResult.nextPaginationToken == null) { - break; - } - usersResult = AuthRecipe.getUsers(main, 500, "ASC", usersResult.nextPaginationToken, - null, null); + + } catch (Exception e) { + throw new RuntimeException(e); + } + }); } es.shutdown(); @@ -426,9 +385,9 @@ private void createSessions(Main main) throws Exception { @Test public void testCreatingOneMillionUsers() throws Exception { - if (System.getenv("ONE_MILLION_USERS_TEST") == null) { - return; - } +// if (System.getenv("ONE_MILLION_USERS_TEST") == null) { +// return; +// } String[] args = {"../"}; TestingProcessManager.TestingProcess process = TestingProcessManager.start(args, false); @@ -486,8 +445,22 @@ public void testCreatingOneMillionUsers() throws Exception { sanityCheckAPIs(process.getProcess()); Runtime.getRuntime().gc(); + System.gc(); + System.runFinalization(); Thread.sleep(10000); + process.kill(false); + process = TestingProcessManager.start(args, false); + Utils.setValueInConfig("firebase_password_hashing_signer_key", + "gRhC3eDeQOdyEn4bMd9c6kxguWVmcIVq/SKa0JDPFeM6TcEevkaW56sIWfx88OHbJKnCXdWscZx0l2WbCJ1wbg=="); + Utils.setValueInConfig("postgresql_connection_pool_size", "500"); + + FeatureFlagTestContent.getInstance(process.getProcess()) + .setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, new EE_FEATURES[]{ + EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY}); + process.startProcess(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Thread memoryChecker = new Thread(() -> { while (memoryCheckRunning.get()) { Runtime rt = Runtime.getRuntime(); @@ -514,7 +487,7 @@ public void testCreatingOneMillionUsers() throws Exception { memoryChecker.join(); System.out.println("Max memory used: " + (maxMemory.get() / (1024 * 1024)) + " MB"); - assert maxMemory.get() < 320 * 1024 * 1024; // must be less than 320 mb + assert maxMemory.get() < 256 * 1024 * 1024; // must be less than 320 mb process.kill(); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); From 8f4b1a9f804ad9af8cc7bd8a706e9999b611924b Mon Sep 17 00:00:00 2001 From: Sattvik Chakravarthy Date: Wed, 13 Mar 2024 17:48:35 +0530 Subject: [PATCH 3/5] fix: versions --- CHANGELOG.md | 2 ++ build.gradle | 2 +- pluginInterfaceSupported.json | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 722bad31..91d99ba2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [7.0.0] + - Replace `TotpNotEnabledError` with `UnknownUserIdTotpError`. - Support for MFA recipe - Adds a new `useStaticKey` param to `updateSessionInfo_Transaction` diff --git a/build.gradle b/build.gradle index baafed34..3d976b54 100644 --- a/build.gradle +++ b/build.gradle @@ -2,7 +2,7 @@ plugins { id 'java-library' } -version = "6.0.0" +version = "7.0.0" repositories { mavenCentral() diff --git a/pluginInterfaceSupported.json b/pluginInterfaceSupported.json index e9d4c148..f9d5be77 100644 --- a/pluginInterfaceSupported.json +++ b/pluginInterfaceSupported.json @@ -1,6 +1,6 @@ { "_comment": "contains a list of plugin interfaces branch names that this core supports", "versions": [ - "5.0" + "6.0" ] } \ No newline at end of file From 68cb4924b3135f98959094e0c506f3522c8dd7cd Mon Sep 17 00:00:00 2001 From: Sattvik Chakravarthy Date: Wed, 13 Mar 2024 17:48:58 +0530 Subject: [PATCH 4/5] fix: versions --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91d99ba2..8322cadf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] -## [7.0.0] +## [7.0.0] - 2024-03-13 - Replace `TotpNotEnabledError` with `UnknownUserIdTotpError`. - Support for MFA recipe From c91dea71c8d3f7cc84f336f0ba2aafc8ce1449c4 Mon Sep 17 00:00:00 2001 From: Sattvik Chakravarthy Date: Wed, 13 Mar 2024 20:20:10 +0530 Subject: [PATCH 5/5] fix: one million users --- .../postgresql/test/OneMillionUsersTest.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/test/java/io/supertokens/storage/postgresql/test/OneMillionUsersTest.java b/src/test/java/io/supertokens/storage/postgresql/test/OneMillionUsersTest.java index 1fd1cc1c..5f002be2 100644 --- a/src/test/java/io/supertokens/storage/postgresql/test/OneMillionUsersTest.java +++ b/src/test/java/io/supertokens/storage/postgresql/test/OneMillionUsersTest.java @@ -385,9 +385,9 @@ private void createSessions(Main main) throws Exception { @Test public void testCreatingOneMillionUsers() throws Exception { -// if (System.getenv("ONE_MILLION_USERS_TEST") == null) { -// return; -// } + if (System.getenv("ONE_MILLION_USERS_TEST") == null) { + return; + } String[] args = {"../"}; TestingProcessManager.TestingProcess process = TestingProcessManager.start(args, false); @@ -443,13 +443,18 @@ public void testCreatingOneMillionUsers() throws Exception { } sanityCheckAPIs(process.getProcess()); + allUserIds.clear(); + allPrimaryUserIds.clear(); + userIdMappings.clear(); + primaryUserIdMappings.clear(); + + process.kill(false); Runtime.getRuntime().gc(); System.gc(); System.runFinalization(); Thread.sleep(10000); - - process.kill(false); + process = TestingProcessManager.start(args, false); Utils.setValueInConfig("firebase_password_hashing_signer_key", "gRhC3eDeQOdyEn4bMd9c6kxguWVmcIVq/SKa0JDPFeM6TcEevkaW56sIWfx88OHbJKnCXdWscZx0l2WbCJ1wbg=="); @@ -487,7 +492,7 @@ public void testCreatingOneMillionUsers() throws Exception { memoryChecker.join(); System.out.println("Max memory used: " + (maxMemory.get() / (1024 * 1024)) + " MB"); - assert maxMemory.get() < 256 * 1024 * 1024; // must be less than 320 mb + assert maxMemory.get() < 256 * 1024 * 1024; // must be less than 256 mb process.kill(); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED));