Merge branch '7.2' into feat/oauth-provider-base

supertokens · Oct 4, 2024 · 0ad059b · 0ad059b
2 parents ff6ec15 + e40602c
commit 0ad059b
Show file tree

Hide file tree

Showing 6 changed files with 122 additions and 5 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,20 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 - Compatible with plugin interface version 6.3
 - Adds support for OAuthStorage
 
+## [7.1.3] - 2024-09-04
+
+- Adds index on `last_active_time` for `user_last_active` table to improve the performance of MAU computation.
+
+### Migration
+
+```sql
+CREATE INDEX IF NOT EXISTS user_last_active_last_active_time_index ON user_last_active (last_active_time DESC, app_id DESC);
+```
+
+## [7.1.2] - 2024-09-02
+
+- Optimizes users count query
+
 ## [7.1.1] - 2024-08-08
 
 - Fixes tests that check for `Internal Error` in 500 status responses

diff --git a/jar/postgresql-plugin-7.1.1.jar → jar/postgresql-plugin-7.1.3.jar b/jar/postgresql-plugin-7.1.1.jar → jar/postgresql-plugin-7.1.3.jar
diff --git a/src/main/java/io/supertokens/storage/postgresql/Start.java b/src/main/java/io/supertokens/storage/postgresql/Start.java
@@ -1338,6 +1338,15 @@ public void updateLastActive(AppIdentifier appIdentifier, String userId) throws
         }
     }
 
+    @TestOnly
+    public void updateLastActive(AppIdentifier appIdentifier, String userId, long timestamp) throws StorageQueryException {
+        try {
+            ActiveUsersQueries.updateUserLastActive(this, appIdentifier, userId, timestamp);
+        } catch (SQLException e) {
+            throw new StorageQueryException(e);
+        }
+    }
+
     @Override
     public int countUsersActiveSince(AppIdentifier appIdentifier, long time) throws StorageQueryException {
         try {

diff --git a/src/main/java/io/supertokens/storage/postgresql/queries/ActiveUsersQueries.java b/src/main/java/io/supertokens/storage/postgresql/queries/ActiveUsersQueries.java
@@ -9,6 +9,8 @@
 import java.sql.Connection;
 import java.sql.SQLException;
 
+import org.jetbrains.annotations.TestOnly;
+
 import static io.supertokens.storage.postgresql.QueryExecutorTemplate.execute;
 import static io.supertokens.storage.postgresql.QueryExecutorTemplate.update;
 import static io.supertokens.storage.postgresql.config.Config.getConfig;
@@ -34,6 +36,11 @@ static String getQueryToCreateAppIdIndexForUserLastActiveTable(Start start) {
                 + Config.getConfig(start).getUserLastActiveTable() + "(app_id);";
     }
 
+    public static String getQueryToCreateLastActiveTimeIndexForUserLastActiveTable(Start start) {
+        return "CREATE INDEX IF NOT EXISTS user_last_active_last_active_time_index ON "
+                + Config.getConfig(start).getUserLastActiveTable() + "(last_active_time DESC, app_id DESC);";
+    }
+
     public static int countUsersActiveSince(Start start, AppIdentifier appIdentifier, long sinceTime)
             throws SQLException, StorageQueryException {
         String QUERY = "SELECT COUNT(*) as total FROM " + Config.getConfig(start).getUserLastActiveTable()
@@ -90,6 +97,22 @@ public static int updateUserLastActive(Start start, AppIdentifier appIdentifier,
         });
     }
 
+    @TestOnly
+    public static int updateUserLastActive(Start start, AppIdentifier appIdentifier, String userId, long timestamp)
+            throws SQLException, StorageQueryException {
+        String QUERY = "INSERT INTO " + Config.getConfig(start).getUserLastActiveTable()
+                +
+                "(app_id, user_id, last_active_time) VALUES(?, ?, ?) ON CONFLICT(app_id, user_id) DO UPDATE SET " +
+                "last_active_time = ?";
+
+        return update(start, QUERY, pst -> {
+            pst.setString(1, appIdentifier.getAppId());
+            pst.setString(2, userId);
+            pst.setLong(3, timestamp);
+            pst.setLong(4, timestamp);
+        });
+    }
+
     public static Long getLastActiveByUserId(Start start, AppIdentifier appIdentifier, String userId)
             throws StorageQueryException {
         String QUERY = "SELECT last_active_time FROM " + Config.getConfig(start).getUserLastActiveTable()

diff --git a/src/main/java/io/supertokens/storage/postgresql/queries/GeneralQueries.java b/src/main/java/io/supertokens/storage/postgresql/queries/GeneralQueries.java
@@ -303,6 +303,8 @@ public static void createTablesIfNotExists(Start start, Connection con) throws S
                     // Index
                     update(con, ActiveUsersQueries.getQueryToCreateAppIdIndexForUserLastActiveTable(start),
                             NO_OP_SETTER);
+                    update(con, ActiveUsersQueries.getQueryToCreateLastActiveTimeIndexForUserLastActiveTable(start),
+                            NO_OP_SETTER);
                 }
 
                 if (!doesTableExists(start, con, Config.getConfig(start).getAccessTokenSigningKeysTable())) {
@@ -740,8 +742,8 @@ public static void deleteKeyValue_Transaction(Start start, Connection con, Tenan
     public static long getUsersCount(Start start, AppIdentifier appIdentifier, RECIPE_ID[] includeRecipeIds)
             throws SQLException, StorageQueryException {
         StringBuilder QUERY = new StringBuilder(
-                "SELECT COUNT(DISTINCT primary_or_recipe_user_id) AS total FROM " +
-                        getConfig(start).getUsersTable());
+                "SELECT COUNT(*) AS total FROM (");
+        QUERY.append("SELECT primary_or_recipe_user_id FROM " + getConfig(start).getUsersTable());
         QUERY.append(" WHERE app_id = ?");
         if (includeRecipeIds != null && includeRecipeIds.length > 0) {
             QUERY.append(" AND recipe_id IN (");
@@ -754,6 +756,7 @@ public static long getUsersCount(Start start, AppIdentifier appIdentifier, RECIP
             }
             QUERY.append(")");
         }
+        QUERY.append(" GROUP BY primary_or_recipe_user_id) AS uniq_users");
 
         return execute(start, QUERY.toString(), pst -> {
             pst.setString(1, appIdentifier.getAppId());
@@ -774,7 +777,8 @@ public static long getUsersCount(Start start, AppIdentifier appIdentifier, RECIP
     public static long getUsersCount(Start start, TenantIdentifier tenantIdentifier, RECIPE_ID[] includeRecipeIds)
             throws SQLException, StorageQueryException {
         StringBuilder QUERY = new StringBuilder(
-                "SELECT COUNT(DISTINCT primary_or_recipe_user_id) AS total FROM " + getConfig(start).getUsersTable());
+                "SELECT COUNT(*) AS total FROM (");
+        QUERY.append("SELECT primary_or_recipe_user_id FROM " + getConfig(start).getUsersTable());
         QUERY.append(" WHERE app_id = ? AND tenant_id = ?");
         if (includeRecipeIds != null && includeRecipeIds.length > 0) {
             QUERY.append(" AND recipe_id IN (");
@@ -788,6 +792,8 @@ public static long getUsersCount(Start start, TenantIdentifier tenantIdentifier,
             QUERY.append(")");
         }
 
+        QUERY.append(" GROUP BY primary_or_recipe_user_id) AS uniq_users");
+
         return execute(start, QUERY.toString(), pst -> {
             pst.setString(1, tenantIdentifier.getAppId());
             pst.setString(2, tenantIdentifier.getTenantId());

diff --git a/src/test/java/io/supertokens/storage/postgresql/test/OneMillionUsersTest.java b/src/test/java/io/supertokens/storage/postgresql/test/OneMillionUsersTest.java
@@ -27,8 +27,11 @@
 import io.supertokens.emailpassword.EmailPassword;
 import io.supertokens.emailpassword.ParsedFirebaseSCryptResponse;
 import io.supertokens.featureflag.EE_FEATURES;
+import io.supertokens.featureflag.FeatureFlag;
 import io.supertokens.featureflag.FeatureFlagTestContent;
 import io.supertokens.passwordless.Passwordless;
+import io.supertokens.pluginInterface.RECIPE_ID;
+import io.supertokens.pluginInterface.Storage;
 import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo;
 import io.supertokens.pluginInterface.authRecipe.LoginMethod;
 import io.supertokens.pluginInterface.authRecipe.sqlStorage.AuthRecipeSQLStorage;
@@ -39,6 +42,7 @@
 import io.supertokens.pluginInterface.thirdparty.sqlStorage.ThirdPartySQLStorage;
 import io.supertokens.session.Session;
 import io.supertokens.session.info.SessionInformationHolder;
+import io.supertokens.storage.postgresql.Start;
 import io.supertokens.storage.postgresql.test.httpRequest.HttpRequestForTesting;
 import io.supertokens.storageLayer.StorageLayer;
 import io.supertokens.thirdparty.ThirdParty;
@@ -386,6 +390,30 @@ private void createSessions(Main main) throws Exception {
         es.awaitTermination(10, TimeUnit.MINUTES);
     }
 
+    private void createActiveUserEntries(Main main) throws Exception {
+        System.out.println("Creating active user entries...");
+
+        ExecutorService es = Executors.newFixedThreadPool(NUM_THREADS);
+
+        for (String userId : allPrimaryUserIds) {
+            String finalUserId = userId;
+            es.execute(() -> {
+                try {
+                    Storage storage = StorageLayer.getBaseStorage(main);
+                    Start start = (Start) storage;
+
+                    start.updateLastActive(new AppIdentifier(null, null), finalUserId, System.currentTimeMillis() - new Random().nextInt(1000 * 3600 * 24 * 60));
+
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+            });
+        }
+
+        es.shutdown();
+        es.awaitTermination(10, TimeUnit.MINUTES);
+    }
+
     @Test
     public void testCreatingOneMillionUsers() throws Exception {
         if (System.getenv("ONE_MILLION_USERS_TEST") == null) {
@@ -400,7 +428,7 @@ public void testCreatingOneMillionUsers() throws Exception {
 
         FeatureFlagTestContent.getInstance(process.getProcess())
                 .setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, new EE_FEATURES[]{
-                        EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY});
+                        EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.DASHBOARD_LOGIN});
         process.startProcess();
         assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED));
 
@@ -445,6 +473,13 @@ public void testCreatingOneMillionUsers() throws Exception {
             System.out.println("Time taken to create sessions: " + ((en - st) / 1000) + " sec");
         }
 
+        {
+            long st = System.currentTimeMillis();
+            createActiveUserEntries(process.getProcess());
+            long en = System.currentTimeMillis();
+            System.out.println("Time taken to create active user entries: " + ((en - st) / 1000) + " sec");
+        }
+
         sanityCheckAPIs(process.getProcess());
         allUserIds.clear();
         allPrimaryUserIds.clear();
@@ -466,7 +501,7 @@ public void testCreatingOneMillionUsers() throws Exception {
 
         FeatureFlagTestContent.getInstance(process.getProcess())
                 .setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, new EE_FEATURES[]{
-                        EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY});
+                        EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.DASHBOARD_LOGIN});
         process.startProcess();
         assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED));
 
@@ -888,6 +923,36 @@ private void measureOperations(Main main) throws Exception {
                 return null;
             });
             System.out.println("Update user metadata " + time);
+            assert time < 3000;
+        }
+
+        { // measure user counting
+            long time = measureTime(() -> {
+                try {
+                    AuthRecipe.getUsersCount(main, null);
+                    AuthRecipe.getUsersCount(main, new RECIPE_ID[]{RECIPE_ID.EMAIL_PASSWORD});
+                    AuthRecipe.getUsersCount(main, new RECIPE_ID[]{RECIPE_ID.EMAIL_PASSWORD, RECIPE_ID.THIRD_PARTY});
+                } catch (Exception e) {
+                    errorCount.incrementAndGet();
+                    throw new RuntimeException(e);
+                }
+                return null;
+            });
+            System.out.println("User counting: " + time);
+            assert time < 10000;
+        }
+        { // measure telemetry
+            long time = measureTime(() -> {
+                try {
+                    FeatureFlag.getInstance(main).getPaidFeatureStats();
+                } catch (Exception e) {
+                    errorCount.incrementAndGet();
+                    throw new RuntimeException(e);
+                }
+                return null;
+            });
+            System.out.println("Telemetry: " + time);
+            assert time < 6000;
         }
 
         assertEquals(0, errorCount.get());