diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dd2981d..1dd709e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,20 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Compatible with plugin interface version 6.3 - Adds support for OAuthStorage +## [7.1.3] - 2024-09-04 + +- Adds index on `last_active_time` for `user_last_active` table to improve the performance of MAU computation. + +### Migration + +```sql +CREATE INDEX IF NOT EXISTS user_last_active_last_active_time_index ON user_last_active (last_active_time DESC, app_id DESC); +``` + +## [7.1.2] - 2024-09-02 + +- Optimizes users count query + ## [7.1.1] - 2024-08-08 - Fixes tests that check for `Internal Error` in 500 status responses diff --git a/jar/postgresql-plugin-7.1.1.jar b/jar/postgresql-plugin-7.1.3.jar similarity index 70% rename from jar/postgresql-plugin-7.1.1.jar rename to jar/postgresql-plugin-7.1.3.jar index 541e501b..af33d925 100644 Binary files a/jar/postgresql-plugin-7.1.1.jar and b/jar/postgresql-plugin-7.1.3.jar differ diff --git a/src/main/java/io/supertokens/storage/postgresql/Start.java b/src/main/java/io/supertokens/storage/postgresql/Start.java index 986e51e0..33cd8c17 100644 --- a/src/main/java/io/supertokens/storage/postgresql/Start.java +++ b/src/main/java/io/supertokens/storage/postgresql/Start.java @@ -1338,6 +1338,15 @@ public void updateLastActive(AppIdentifier appIdentifier, String userId) throws } } + @TestOnly + public void updateLastActive(AppIdentifier appIdentifier, String userId, long timestamp) throws StorageQueryException { + try { + ActiveUsersQueries.updateUserLastActive(this, appIdentifier, userId, timestamp); + } catch (SQLException e) { + throw new StorageQueryException(e); + } + } + @Override public int countUsersActiveSince(AppIdentifier appIdentifier, long time) throws StorageQueryException { try { diff --git a/src/main/java/io/supertokens/storage/postgresql/queries/ActiveUsersQueries.java b/src/main/java/io/supertokens/storage/postgresql/queries/ActiveUsersQueries.java index 327ed6ce..ccd589ac 100644 --- a/src/main/java/io/supertokens/storage/postgresql/queries/ActiveUsersQueries.java +++ b/src/main/java/io/supertokens/storage/postgresql/queries/ActiveUsersQueries.java @@ -9,6 +9,8 @@ import java.sql.Connection; import java.sql.SQLException; +import org.jetbrains.annotations.TestOnly; + import static io.supertokens.storage.postgresql.QueryExecutorTemplate.execute; import static io.supertokens.storage.postgresql.QueryExecutorTemplate.update; import static io.supertokens.storage.postgresql.config.Config.getConfig; @@ -34,6 +36,11 @@ static String getQueryToCreateAppIdIndexForUserLastActiveTable(Start start) { + Config.getConfig(start).getUserLastActiveTable() + "(app_id);"; } + public static String getQueryToCreateLastActiveTimeIndexForUserLastActiveTable(Start start) { + return "CREATE INDEX IF NOT EXISTS user_last_active_last_active_time_index ON " + + Config.getConfig(start).getUserLastActiveTable() + "(last_active_time DESC, app_id DESC);"; + } + public static int countUsersActiveSince(Start start, AppIdentifier appIdentifier, long sinceTime) throws SQLException, StorageQueryException { String QUERY = "SELECT COUNT(*) as total FROM " + Config.getConfig(start).getUserLastActiveTable() @@ -90,6 +97,22 @@ public static int updateUserLastActive(Start start, AppIdentifier appIdentifier, }); } + @TestOnly + public static int updateUserLastActive(Start start, AppIdentifier appIdentifier, String userId, long timestamp) + throws SQLException, StorageQueryException { + String QUERY = "INSERT INTO " + Config.getConfig(start).getUserLastActiveTable() + + + "(app_id, user_id, last_active_time) VALUES(?, ?, ?) ON CONFLICT(app_id, user_id) DO UPDATE SET " + + "last_active_time = ?"; + + return update(start, QUERY, pst -> { + pst.setString(1, appIdentifier.getAppId()); + pst.setString(2, userId); + pst.setLong(3, timestamp); + pst.setLong(4, timestamp); + }); + } + public static Long getLastActiveByUserId(Start start, AppIdentifier appIdentifier, String userId) throws StorageQueryException { String QUERY = "SELECT last_active_time FROM " + Config.getConfig(start).getUserLastActiveTable() diff --git a/src/main/java/io/supertokens/storage/postgresql/queries/GeneralQueries.java b/src/main/java/io/supertokens/storage/postgresql/queries/GeneralQueries.java index 7eed849c..a53f0bdd 100644 --- a/src/main/java/io/supertokens/storage/postgresql/queries/GeneralQueries.java +++ b/src/main/java/io/supertokens/storage/postgresql/queries/GeneralQueries.java @@ -303,6 +303,8 @@ public static void createTablesIfNotExists(Start start, Connection con) throws S // Index update(con, ActiveUsersQueries.getQueryToCreateAppIdIndexForUserLastActiveTable(start), NO_OP_SETTER); + update(con, ActiveUsersQueries.getQueryToCreateLastActiveTimeIndexForUserLastActiveTable(start), + NO_OP_SETTER); } if (!doesTableExists(start, con, Config.getConfig(start).getAccessTokenSigningKeysTable())) { @@ -740,8 +742,8 @@ public static void deleteKeyValue_Transaction(Start start, Connection con, Tenan public static long getUsersCount(Start start, AppIdentifier appIdentifier, RECIPE_ID[] includeRecipeIds) throws SQLException, StorageQueryException { StringBuilder QUERY = new StringBuilder( - "SELECT COUNT(DISTINCT primary_or_recipe_user_id) AS total FROM " + - getConfig(start).getUsersTable()); + "SELECT COUNT(*) AS total FROM ("); + QUERY.append("SELECT primary_or_recipe_user_id FROM " + getConfig(start).getUsersTable()); QUERY.append(" WHERE app_id = ?"); if (includeRecipeIds != null && includeRecipeIds.length > 0) { QUERY.append(" AND recipe_id IN ("); @@ -754,6 +756,7 @@ public static long getUsersCount(Start start, AppIdentifier appIdentifier, RECIP } QUERY.append(")"); } + QUERY.append(" GROUP BY primary_or_recipe_user_id) AS uniq_users"); return execute(start, QUERY.toString(), pst -> { pst.setString(1, appIdentifier.getAppId()); @@ -774,7 +777,8 @@ public static long getUsersCount(Start start, AppIdentifier appIdentifier, RECIP public static long getUsersCount(Start start, TenantIdentifier tenantIdentifier, RECIPE_ID[] includeRecipeIds) throws SQLException, StorageQueryException { StringBuilder QUERY = new StringBuilder( - "SELECT COUNT(DISTINCT primary_or_recipe_user_id) AS total FROM " + getConfig(start).getUsersTable()); + "SELECT COUNT(*) AS total FROM ("); + QUERY.append("SELECT primary_or_recipe_user_id FROM " + getConfig(start).getUsersTable()); QUERY.append(" WHERE app_id = ? AND tenant_id = ?"); if (includeRecipeIds != null && includeRecipeIds.length > 0) { QUERY.append(" AND recipe_id IN ("); @@ -788,6 +792,8 @@ public static long getUsersCount(Start start, TenantIdentifier tenantIdentifier, QUERY.append(")"); } + QUERY.append(" GROUP BY primary_or_recipe_user_id) AS uniq_users"); + return execute(start, QUERY.toString(), pst -> { pst.setString(1, tenantIdentifier.getAppId()); pst.setString(2, tenantIdentifier.getTenantId()); diff --git a/src/test/java/io/supertokens/storage/postgresql/test/OneMillionUsersTest.java b/src/test/java/io/supertokens/storage/postgresql/test/OneMillionUsersTest.java index 5f55bcf1..ea29a8f9 100644 --- a/src/test/java/io/supertokens/storage/postgresql/test/OneMillionUsersTest.java +++ b/src/test/java/io/supertokens/storage/postgresql/test/OneMillionUsersTest.java @@ -27,8 +27,11 @@ import io.supertokens.emailpassword.EmailPassword; import io.supertokens.emailpassword.ParsedFirebaseSCryptResponse; import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlag; import io.supertokens.featureflag.FeatureFlagTestContent; import io.supertokens.passwordless.Passwordless; +import io.supertokens.pluginInterface.RECIPE_ID; +import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; import io.supertokens.pluginInterface.authRecipe.LoginMethod; import io.supertokens.pluginInterface.authRecipe.sqlStorage.AuthRecipeSQLStorage; @@ -39,6 +42,7 @@ import io.supertokens.pluginInterface.thirdparty.sqlStorage.ThirdPartySQLStorage; import io.supertokens.session.Session; import io.supertokens.session.info.SessionInformationHolder; +import io.supertokens.storage.postgresql.Start; import io.supertokens.storage.postgresql.test.httpRequest.HttpRequestForTesting; import io.supertokens.storageLayer.StorageLayer; import io.supertokens.thirdparty.ThirdParty; @@ -386,6 +390,30 @@ private void createSessions(Main main) throws Exception { es.awaitTermination(10, TimeUnit.MINUTES); } + private void createActiveUserEntries(Main main) throws Exception { + System.out.println("Creating active user entries..."); + + ExecutorService es = Executors.newFixedThreadPool(NUM_THREADS); + + for (String userId : allPrimaryUserIds) { + String finalUserId = userId; + es.execute(() -> { + try { + Storage storage = StorageLayer.getBaseStorage(main); + Start start = (Start) storage; + + start.updateLastActive(new AppIdentifier(null, null), finalUserId, System.currentTimeMillis() - new Random().nextInt(1000 * 3600 * 24 * 60)); + + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + es.shutdown(); + es.awaitTermination(10, TimeUnit.MINUTES); + } + @Test public void testCreatingOneMillionUsers() throws Exception { if (System.getenv("ONE_MILLION_USERS_TEST") == null) { @@ -400,7 +428,7 @@ public void testCreatingOneMillionUsers() throws Exception { FeatureFlagTestContent.getInstance(process.getProcess()) .setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, new EE_FEATURES[]{ - EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY}); + EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.DASHBOARD_LOGIN}); process.startProcess(); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); @@ -445,6 +473,13 @@ public void testCreatingOneMillionUsers() throws Exception { System.out.println("Time taken to create sessions: " + ((en - st) / 1000) + " sec"); } + { + long st = System.currentTimeMillis(); + createActiveUserEntries(process.getProcess()); + long en = System.currentTimeMillis(); + System.out.println("Time taken to create active user entries: " + ((en - st) / 1000) + " sec"); + } + sanityCheckAPIs(process.getProcess()); allUserIds.clear(); allPrimaryUserIds.clear(); @@ -466,7 +501,7 @@ public void testCreatingOneMillionUsers() throws Exception { FeatureFlagTestContent.getInstance(process.getProcess()) .setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, new EE_FEATURES[]{ - EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY}); + EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.DASHBOARD_LOGIN}); process.startProcess(); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); @@ -888,6 +923,36 @@ private void measureOperations(Main main) throws Exception { return null; }); System.out.println("Update user metadata " + time); + assert time < 3000; + } + + { // measure user counting + long time = measureTime(() -> { + try { + AuthRecipe.getUsersCount(main, null); + AuthRecipe.getUsersCount(main, new RECIPE_ID[]{RECIPE_ID.EMAIL_PASSWORD}); + AuthRecipe.getUsersCount(main, new RECIPE_ID[]{RECIPE_ID.EMAIL_PASSWORD, RECIPE_ID.THIRD_PARTY}); + } catch (Exception e) { + errorCount.incrementAndGet(); + throw new RuntimeException(e); + } + return null; + }); + System.out.println("User counting: " + time); + assert time < 10000; + } + { // measure telemetry + long time = measureTime(() -> { + try { + FeatureFlag.getInstance(main).getPaidFeatureStats(); + } catch (Exception e) { + errorCount.incrementAndGet(); + throw new RuntimeException(e); + } + return null; + }); + System.out.println("Telemetry: " + time); + assert time < 6000; } assertEquals(0, errorCount.get());