Fix download stats, restore community activity

These changes should:
- Have better download statistics, both for the total amounts, as for the per-project amounts
- Restore the community activity sidebar to show recent (last 7 day) downloads, posts and member activity

This functionality requires a couple of queries to be set up in Discourse's Data Explorer (which I've already done) and a Discourse API key to be configured in Tomcat's context (which I've also already configured).

src/it/tomcat8x/context.xml

@@ -18,6 +18,9 @@
     <Parameter name="openfire-plugins-beta-path" value="${catalina.base}/staticfiles/builds/openfire/plugins-beta"/>
     <Parameter name="openfire-plugins-dev-path"  value="${catalina.base}/staticfiles/builds/openfire/plugins-dev"/>
 
+    <!-- Used to interact with Discourse's API -->
+    <Parameter name="discourse-api-key" value="CHANGEME"/>
+
     <!-- WEB-50: JNDI datasource for the database that the website uses to collect statistics. -->
     <Resource
         name="jdbc/stats"

src/main/java/org/jivesoftware/site/DiscourseAPI.java

@@ -0,0 +1,126 @@
+package org.jivesoftware.site;
+
+import org.jivesoftware.webservices.RestClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import java.util.HashMap;
+import java.util.Hashtable;
+import java.util.Map;
+
+public class DiscourseAPI extends HttpServlet
+{
+    private static final Logger Log = LoggerFactory.getLogger(DiscourseAPI.class);
+
+    private static long CACHE_PERIOD = 30 * 60 * 1000; // 30 minutes
+
+    private static long lastUpdate = 0;
+
+    private static Map<Integer, Long> counts = new Hashtable<>();
+
+    private static String baseUrl;
+    private static String apiKey;
+    private static RestClient restClient;
+
+    public void init(ServletConfig servletConfig) throws ServletException
+    {
+        super.init(servletConfig);
+
+        baseUrl = servletConfig.getServletContext().getInitParameter("discourse_baseurl");
+        if ( baseUrl == null || baseUrl.isEmpty() )
+        {
+            baseUrl = "https://discourse.igniterealtime.org/";
+        }
+
+        apiKey = servletConfig.getServletContext().getInitParameter("discourse-api-key");
+
+        restClient = new RestClient();
+    }
+
+    public static Long getActiveMembersLast7Days()
+    {
+        collectTotals();
+        if (counts.containsKey(3)) {
+            return counts.get(3);
+        }
+        return null;
+    }
+
+    public static Long getNewPostsLast7Days()
+    {
+        collectTotals();
+        if (counts.containsKey(4)) {
+            return counts.get(4);
+        }
+        return null;
+    }
+
+    /**
+     * Executes a query that's defined in Discourse's Data Explorer. The query is assumed to have a singular, numeric
+     * return value.
+     *
+     * @param queryId Discourse Data Explorer Query identifier
+     * @param lastNumberOfDays The value for the 'duration_days' parameter
+     * @return the query result, or null if an exception occurred.
+     */
+    private static Long doSimpleQuery(final int queryId, final int lastNumberOfDays) {
+        final Map<String, String> headers = new HashMap<>();
+        headers.put("Api-Key", apiKey);
+        headers.put("Api-Username", "system");
+        final Map<String, String> parameters = new HashMap<>();
+        parameters.put("params", "{\"duration_days\":\""+lastNumberOfDays+"\"}");
+        parameters.put("explain", "false");
+        parameters.put("download", "true");
+
+        try {
+            return restClient.post(baseUrl + "/admin/plugins/explorer/queries/"+queryId+"/run", headers, parameters).getJSONArray("rows").getJSONArray(0).getLong(0);
+        } catch (Throwable t) {
+            Log.warn("Unable to interact with Discourse's API.", t);
+            return null;
+        }
+    }
+
+    /**
+     * Collects all of the totals from the API. Has a rudimentary caching mechanism
+     * so that the queries are only run every CACHE_PERIOD milliseconds.
+     */
+    private synchronized static void collectTotals() {
+        // See if we need to update the totals
+        if ((lastUpdate + CACHE_PERIOD) > System.currentTimeMillis()) {
+            return;
+        }
+        lastUpdate = System.currentTimeMillis();
+
+        // Collect the new totals on a background thread since they could take a while
+        Thread collectorThread = new Thread(new DiscourseAPI.DownloadStatsRunnable(counts));
+        collectorThread.start();
+        if (counts.isEmpty()) {
+            // Need to wait for the collectorThread to finish since the counts are not initialized yet
+            try {
+                collectorThread.join();
+            }
+            catch (Exception e) { Log.debug( "An exception occurred that can probably be ignored.", e); }
+        }
+    }
+
+    private static class DownloadStatsRunnable implements Runnable {
+        private Map<Integer, Long> counts;
+
+        public DownloadStatsRunnable(Map<Integer, Long> counts) {
+            this.counts = counts;
+        }
+
+        public void run() {
+            final Map<Integer, Long> results = new HashMap<>();
+            results.put(3, doSimpleQuery(3, 7));
+            results.put(4, doSimpleQuery(4, 7));
+
+            // Replace all values in the object used by the website in one go.
+            counts.clear();
+            counts.putAll(results);
+        }
+    }
+}

src/main/java/org/jivesoftware/site/DownloadServlet.java

@@ -58,6 +58,24 @@ public String getName() {
             return name;
         }
 
+        public static DownloadInfo getDownloadInfo(int type)
+        {
+            switch (type) {
+                case 0: return openfire;
+                case 1: return spark;
+                case 2: return smack;
+                case 3: return xiff;
+                case 4: return spark_update;
+                case 5: return openfire_plugin;
+                case 6: return spark_plugin;
+                case 7: return wildfire;
+                case 8: return wildfire_plugin;
+                case 9: return whack;
+                case 10: return sparkweb;
+                case 11: return tinder;
+                default: return null;
+            }
+        }
     }
 
     public void doGet(HttpServletRequest request, HttpServletResponse response) throws

src/main/java/org/jivesoftware/site/DownloadStats.java

@@ -37,11 +37,11 @@ public class DownloadStats extends HttpServlet {
     // SQL for inserting the update info check
     private static final String ADD_UPDATE_INFO = "insert into checkUpdateInfo (ipAddress, os, type, time, country, region, city, currentVersion, latestVersion) values (INET_ATON(?),?,?,NOW(),?,?,?,?,?)";
 
-    // SQL for counting the total number of downloads
-    private static String COUNT_TOTAL_DOWNLOADS = "SELECT count(*) FROM downloadInfo";
+    // SQL for counting the total number of downloads by type.
+    private static String COUNT_TOTAL_DOWNLOADS_BY_TYPE = "SELECT type, count(type) FROM downloadInfo GROUP BY type";
 
-    // SQL for counting the total number of downloads for a particular download type
-    private static String COUNT_TOTAL_DOWNLOADS_FOR_TYPE = "SELECT count(*) FROM downloadInfo WHERE type = ?";
+    // SQL for counting the total number of downloads in the last 7 days.
+    private static String COUNT_TOTAL_DOWNLOADS_LAST_7_DAYS = "SELECT count(*) FROM downloadInfo WHERE time >= DATE(NOW() - INTERVAL 7 DAY)";
 
     // Period between cache updates
     private static long CACHE_PERIOD = 30 * 60 * 1000; // 30 minutes
@@ -50,8 +50,9 @@ public class DownloadStats extends HttpServlet {
     private static long lastUpdate = 0;
 
     // Using a Hashtable for synchronization
-    private static Map<String, Long> counts = new Hashtable<String, Long>();
+    private static Map<String, Long> counts = new Hashtable<>();
     private static final String TOTAL = "total";
+    private static final String TOTAL7DAYS = "total7days";
 
     // A reference to the builds directory
     private static File buildsDirectory;
@@ -137,7 +138,23 @@ public static long getDownloadsForType(DownloadServlet.DownloadInfo type) {
      */
     public static long getTotalDownloads() {
         collectTotals();
-        return counts.get(TOTAL);
+        if (counts.containsKey(TOTAL)) {
+            return counts.get(TOTAL);
+        }
+        return 0;
+    }
+
+    /**
+     * Count the total number of downloads in the last 7 days.
+     *
+     * @return the total number of downloads in the last 7 days.
+     */
+    public static long getTotalDownloadsLast7Days() {
+        collectTotals();
+        if (counts.containsKey(TOTAL7DAYS)) {
+            return counts.get(TOTAL7DAYS);
+        }
+        return 0;
     }
 
     /**
@@ -164,7 +181,7 @@ private synchronized static void collectTotals() {
     }
 
     private static class DownloadStatsRunnable implements Runnable {
-        private Map<String, Long> counts = null;
+        private Map<String, Long> counts;
 
         public DownloadStatsRunnable(Map<String, Long> counts) {
             this.counts = counts;
@@ -175,77 +192,78 @@ public void run() {
             Connection con = null;
             PreparedStatement pstmt = null;
             ResultSet rs = null;
-            // The count starts at 1282298 which is an estimate of the number of downloads prior to
-            // accurate download stats being collected. This number was derived by performing a linear
-            // regression from the time the project was first available until November 30, 2006.
-            long count = 1282298L;
 
-            // Get the total count
+            final Map<String, Long> results = new Hashtable<>();
+            long total = 0L;
             try {
                 con = connectionManager.getConnection();
-                pstmt = con.prepareStatement(COUNT_TOTAL_DOWNLOADS);
+                pstmt = con.prepareStatement(COUNT_TOTAL_DOWNLOADS_BY_TYPE);
                 rs = pstmt.executeQuery();
+
+                while (rs.next()) {
+                    final int type = rs.getInt(2);
+                    long amount = rs.getLong(1);
+
+                    final DownloadServlet.DownloadInfo downloadInfo = DownloadServlet.DownloadInfo.getDownloadInfo(type);
+                    if (downloadInfo == null) {
+                        continue;
+                    }
+
+                    // The count starts at an estimate of the number of downloads prior to
+                    // accurate download stats being collected. This number was derived by performing a linear
+                    // regression from the time the project was first available until November 30, 2006.
+                    switch (downloadInfo) {
+                        case openfire:
+                            amount += 675774L;
+                            break;
+                        case spark:
+                            amount += 438159L;
+                            break;
+                        case smack:
+                            amount += 332007L;
+                            break;
+                        case xiff:
+                            amount += 4683L;
+                            break;
+                        default:
+                            break;
+                    }
+                    total += amount;
+                    results.put(downloadInfo.getName(), amount);
+                }
+                results.put(TOTAL, total);
+
+                // Combine Openfire and Wildfire results (as they're different historical names for the same project).
+                results.put(DownloadServlet.DownloadInfo.openfire.getName(), results.get(DownloadServlet.DownloadInfo.openfire.getName()) + results.get(DownloadServlet.DownloadInfo.wildfire.getName()));
+                results.remove(DownloadServlet.DownloadInfo.wildfire.getName());
+                results.put(DownloadServlet.DownloadInfo.openfire_plugin.getName(), results.get(DownloadServlet.DownloadInfo.openfire_plugin.getName()) + results.get(DownloadServlet.DownloadInfo.wildfire_plugin.getName()));
+                results.remove(DownloadServlet.DownloadInfo.wildfire_plugin.getName());
+
+                results.forEach((key, value) -> System.out.println(key + ": " + value));
+
+                rs.close();
+                pstmt.close();
+
+                pstmt = con.prepareStatement(COUNT_TOTAL_DOWNLOADS_LAST_7_DAYS);
+                rs = pstmt.executeQuery();
+                long lastDays = 0L;
                 if (rs.next()) {
-                    count += rs.getLong(1);
+                    lastDays = rs.getLong(1);
                 }
-            }
-            catch (Exception e) {
-                Log.warn("Error counting total downloads.", e);
+                results.put(TOTAL7DAYS, lastDays);
+
+                // Replace all values in the object used by the website in one go.
+                counts.clear();
+                counts.putAll(results);
+            } catch (Exception e) {
+                Log.warn("Error counting downloads.", e);
             }
             finally {
                 if (rs != null) {
                     try { rs.close(); } catch (Exception e) { Log.debug( "An exception occurred that can probably be ignored.", e); }
                 }
                 DbConnectionManager.close(pstmt, con);
             }
-            counts.put(TOTAL, count);
-
-            // Get the count for each type
-            for (DownloadServlet.DownloadInfo type : DownloadServlet.DownloadInfo.values()) {
-                // The count starts at an estimate of the number of downloads prior to
-                // accurate download stats being collected. This number was derived by performing a linear
-                // regression from the time the project was first available until November 30, 2006.
-                switch (type) {
-                    case openfire:
-                        count = 675774L;
-                        break;
-                    case spark:
-                        count = 438159L;
-                        break;
-                    case smack:
-                        count = 332007L;
-                        break;
-                    case xiff:
-                        count = 4683L;
-                        break;
-                    default:
-                        count = 0L;
-                }
-
-                try {
-                    con = connectionManager.getConnection();
-                    pstmt = con.prepareStatement(COUNT_TOTAL_DOWNLOADS_FOR_TYPE);
-                    pstmt.setInt(1, type.getType());
-                    rs = pstmt.executeQuery();
-                    if (rs.next()) {
-                        count += rs.getLong(1);
-                    }
-                }
-                catch (Exception e) {
-                    String name = null;
-                    if (type != null) {
-                        name = type.getName();
-                    }
-                    Log.warn( "Error counting downloads for type " + name, e);
-                }
-                finally {
-                    if (rs != null) {
-                        try { rs.close(); } catch (Exception e) { Log.debug( "An exception occurred that can probably be ignored.", e); }
-                    }
-                    DbConnectionManager.close(pstmt, con);
-                }
-                counts.put(type.getName(), count);
-            }
         }
     }