From bd8ea124b073e812dce7c8f8c618314b5a0320b6 Mon Sep 17 00:00:00 2001 From: Guus der Kinderen Date: Wed, 18 Sep 2024 21:48:01 +0200 Subject: [PATCH] Fix download stats, restore community activity These changes should: - Have better download statistics, both for the total amounts, as for the per-project amounts - Restore the community activity sidebar to show recent (last 7 day) downloads, posts and member activity This functionality requires a couple of queries to be set up in Discourse's Data Explorer (which I've already done) and a Discourse API key to be configured in Tomcat's context (which I've also already configured). --- src/it/tomcat8x/context.xml | 3 + .../org/jivesoftware/site/DiscourseAPI.java | 126 +++++++++++++++ .../jivesoftware/site/DownloadServlet.java | 18 +++ .../org/jivesoftware/site/DownloadStats.java | 148 ++++++++++-------- .../jivesoftware/webservices/RestClient.java | 34 ++++ src/main/webapp/WEB-INF/web.xml | 5 + src/main/webapp/downloads/beta.jsp | 2 +- src/main/webapp/downloads/index.jsp | 2 +- .../webapp/downloads/nightly_openfire.jsp | 2 +- src/main/webapp/downloads/nightly_smack.jsp | 2 +- src/main/webapp/downloads/nightly_spark.jsp | 2 +- src/main/webapp/downloads/nightly_xiff.jsp | 2 +- src/main/webapp/downloads/source.jsp | 2 +- src/main/webapp/fans/index.jsp | 2 +- .../webapp/includes/sidebar_48hrsnapshot.jspf | 36 ----- .../webapp/includes/sidebar_7daySnapshot.jspf | 37 +++++ src/main/webapp/includes/sidebar_snapshot.jsp | 40 +++-- src/main/webapp/includes/sidebar_support.jspf | 1 + src/main/webapp/index.jsp | 2 - src/main/webapp/news/index.jsp | 14 -- src/main/webapp/projects/index.jsp | 2 +- 21 files changed, 339 insertions(+), 143 deletions(-) create mode 100644 src/main/java/org/jivesoftware/site/DiscourseAPI.java delete mode 100644 src/main/webapp/includes/sidebar_48hrsnapshot.jspf create mode 100644 src/main/webapp/includes/sidebar_7daySnapshot.jspf diff --git a/src/it/tomcat8x/context.xml b/src/it/tomcat8x/context.xml index 50c7bc39..c6dcd98f 100644 --- a/src/it/tomcat8x/context.xml +++ b/src/it/tomcat8x/context.xml @@ -18,6 +18,9 @@ + + + counts = new Hashtable<>(); + + private static String baseUrl; + private static String apiKey; + private static RestClient restClient; + + public void init(ServletConfig servletConfig) throws ServletException + { + super.init(servletConfig); + + baseUrl = servletConfig.getServletContext().getInitParameter("discourse_baseurl"); + if ( baseUrl == null || baseUrl.isEmpty() ) + { + baseUrl = "https://discourse.igniterealtime.org/"; + } + + apiKey = servletConfig.getServletContext().getInitParameter("discourse-api-key"); + + restClient = new RestClient(); + } + + public static Long getActiveMembersLast7Days() + { + collectTotals(); + if (counts.containsKey(3)) { + return counts.get(3); + } + return null; + } + + public static Long getNewPostsLast7Days() + { + collectTotals(); + if (counts.containsKey(4)) { + return counts.get(4); + } + return null; + } + + /** + * Executes a query that's defined in Discourse's Data Explorer. The query is assumed to have a singular, numeric + * return value. + * + * @param queryId Discourse Data Explorer Query identifier + * @param lastNumberOfDays The value for the 'duration_days' parameter + * @return the query result, or null if an exception occurred. + */ + private static Long doSimpleQuery(final int queryId, final int lastNumberOfDays) { + final Map headers = new HashMap<>(); + headers.put("Api-Key", apiKey); + headers.put("Api-Username", "system"); + final Map parameters = new HashMap<>(); + parameters.put("params", "{\"duration_days\":\""+lastNumberOfDays+"\"}"); + parameters.put("explain", "false"); + parameters.put("download", "true"); + + try { + return restClient.post(baseUrl + "/admin/plugins/explorer/queries/"+queryId+"/run", headers, parameters).getJSONArray("rows").getJSONArray(0).getLong(0); + } catch (Throwable t) { + Log.warn("Unable to interact with Discourse's API.", t); + return null; + } + } + + /** + * Collects all of the totals from the API. Has a rudimentary caching mechanism + * so that the queries are only run every CACHE_PERIOD milliseconds. + */ + private synchronized static void collectTotals() { + // See if we need to update the totals + if ((lastUpdate + CACHE_PERIOD) > System.currentTimeMillis()) { + return; + } + lastUpdate = System.currentTimeMillis(); + + // Collect the new totals on a background thread since they could take a while + Thread collectorThread = new Thread(new DiscourseAPI.DownloadStatsRunnable(counts)); + collectorThread.start(); + if (counts.isEmpty()) { + // Need to wait for the collectorThread to finish since the counts are not initialized yet + try { + collectorThread.join(); + } + catch (Exception e) { Log.debug( "An exception occurred that can probably be ignored.", e); } + } + } + + private static class DownloadStatsRunnable implements Runnable { + private Map counts; + + public DownloadStatsRunnable(Map counts) { + this.counts = counts; + } + + public void run() { + final Map results = new HashMap<>(); + results.put(3, doSimpleQuery(3, 7)); + results.put(4, doSimpleQuery(4, 7)); + + // Replace all values in the object used by the website in one go. + counts.clear(); + counts.putAll(results); + } + } +} diff --git a/src/main/java/org/jivesoftware/site/DownloadServlet.java b/src/main/java/org/jivesoftware/site/DownloadServlet.java index a2cca634..5e9f73ca 100644 --- a/src/main/java/org/jivesoftware/site/DownloadServlet.java +++ b/src/main/java/org/jivesoftware/site/DownloadServlet.java @@ -58,6 +58,24 @@ public String getName() { return name; } + public static DownloadInfo getDownloadInfo(int type) + { + switch (type) { + case 0: return openfire; + case 1: return spark; + case 2: return smack; + case 3: return xiff; + case 4: return spark_update; + case 5: return openfire_plugin; + case 6: return spark_plugin; + case 7: return wildfire; + case 8: return wildfire_plugin; + case 9: return whack; + case 10: return sparkweb; + case 11: return tinder; + default: return null; + } + } } public void doGet(HttpServletRequest request, HttpServletResponse response) throws diff --git a/src/main/java/org/jivesoftware/site/DownloadStats.java b/src/main/java/org/jivesoftware/site/DownloadStats.java index a56b274e..82860eef 100644 --- a/src/main/java/org/jivesoftware/site/DownloadStats.java +++ b/src/main/java/org/jivesoftware/site/DownloadStats.java @@ -37,11 +37,11 @@ public class DownloadStats extends HttpServlet { // SQL for inserting the update info check private static final String ADD_UPDATE_INFO = "insert into checkUpdateInfo (ipAddress, os, type, time, country, region, city, currentVersion, latestVersion) values (INET_ATON(?),?,?,NOW(),?,?,?,?,?)"; - // SQL for counting the total number of downloads - private static String COUNT_TOTAL_DOWNLOADS = "SELECT count(*) FROM downloadInfo"; + // SQL for counting the total number of downloads by type. + private static String COUNT_TOTAL_DOWNLOADS_BY_TYPE = "SELECT type, count(type) FROM downloadInfo GROUP BY type"; - // SQL for counting the total number of downloads for a particular download type - private static String COUNT_TOTAL_DOWNLOADS_FOR_TYPE = "SELECT count(*) FROM downloadInfo WHERE type = ?"; + // SQL for counting the total number of downloads in the last 7 days. + private static String COUNT_TOTAL_DOWNLOADS_LAST_7_DAYS = "SELECT count(*) FROM downloadInfo WHERE time >= DATE(NOW() - INTERVAL 7 DAY)"; // Period between cache updates private static long CACHE_PERIOD = 30 * 60 * 1000; // 30 minutes @@ -50,8 +50,9 @@ public class DownloadStats extends HttpServlet { private static long lastUpdate = 0; // Using a Hashtable for synchronization - private static Map counts = new Hashtable(); + private static Map counts = new Hashtable<>(); private static final String TOTAL = "total"; + private static final String TOTAL7DAYS = "total7days"; // A reference to the builds directory private static File buildsDirectory; @@ -137,7 +138,23 @@ public static long getDownloadsForType(DownloadServlet.DownloadInfo type) { */ public static long getTotalDownloads() { collectTotals(); - return counts.get(TOTAL); + if (counts.containsKey(TOTAL)) { + return counts.get(TOTAL); + } + return 0; + } + + /** + * Count the total number of downloads in the last 7 days. + * + * @return the total number of downloads in the last 7 days. + */ + public static long getTotalDownloadsLast7Days() { + collectTotals(); + if (counts.containsKey(TOTAL7DAYS)) { + return counts.get(TOTAL7DAYS); + } + return 0; } /** @@ -164,7 +181,7 @@ private synchronized static void collectTotals() { } private static class DownloadStatsRunnable implements Runnable { - private Map counts = null; + private Map counts; public DownloadStatsRunnable(Map counts) { this.counts = counts; @@ -175,22 +192,71 @@ public void run() { Connection con = null; PreparedStatement pstmt = null; ResultSet rs = null; - // The count starts at 1282298 which is an estimate of the number of downloads prior to - // accurate download stats being collected. This number was derived by performing a linear - // regression from the time the project was first available until November 30, 2006. - long count = 1282298L; - // Get the total count + final Map results = new Hashtable<>(); + long total = 0L; try { con = connectionManager.getConnection(); - pstmt = con.prepareStatement(COUNT_TOTAL_DOWNLOADS); + pstmt = con.prepareStatement(COUNT_TOTAL_DOWNLOADS_BY_TYPE); rs = pstmt.executeQuery(); + + while (rs.next()) { + final int type = rs.getInt(2); + long amount = rs.getLong(1); + + final DownloadServlet.DownloadInfo downloadInfo = DownloadServlet.DownloadInfo.getDownloadInfo(type); + if (downloadInfo == null) { + continue; + } + + // The count starts at an estimate of the number of downloads prior to + // accurate download stats being collected. This number was derived by performing a linear + // regression from the time the project was first available until November 30, 2006. + switch (downloadInfo) { + case openfire: + amount += 675774L; + break; + case spark: + amount += 438159L; + break; + case smack: + amount += 332007L; + break; + case xiff: + amount += 4683L; + break; + default: + break; + } + total += amount; + results.put(downloadInfo.getName(), amount); + } + results.put(TOTAL, total); + + // Combine Openfire and Wildfire results (as they're different historical names for the same project). + results.put(DownloadServlet.DownloadInfo.openfire.getName(), results.get(DownloadServlet.DownloadInfo.openfire.getName()) + results.get(DownloadServlet.DownloadInfo.wildfire.getName())); + results.remove(DownloadServlet.DownloadInfo.wildfire.getName()); + results.put(DownloadServlet.DownloadInfo.openfire_plugin.getName(), results.get(DownloadServlet.DownloadInfo.openfire_plugin.getName()) + results.get(DownloadServlet.DownloadInfo.wildfire_plugin.getName())); + results.remove(DownloadServlet.DownloadInfo.wildfire_plugin.getName()); + + results.forEach((key, value) -> System.out.println(key + ": " + value)); + + rs.close(); + pstmt.close(); + + pstmt = con.prepareStatement(COUNT_TOTAL_DOWNLOADS_LAST_7_DAYS); + rs = pstmt.executeQuery(); + long lastDays = 0L; if (rs.next()) { - count += rs.getLong(1); + lastDays = rs.getLong(1); } - } - catch (Exception e) { - Log.warn("Error counting total downloads.", e); + results.put(TOTAL7DAYS, lastDays); + + // Replace all values in the object used by the website in one go. + counts.clear(); + counts.putAll(results); + } catch (Exception e) { + Log.warn("Error counting downloads.", e); } finally { if (rs != null) { @@ -198,54 +264,6 @@ public void run() { } DbConnectionManager.close(pstmt, con); } - counts.put(TOTAL, count); - - // Get the count for each type - for (DownloadServlet.DownloadInfo type : DownloadServlet.DownloadInfo.values()) { - // The count starts at an estimate of the number of downloads prior to - // accurate download stats being collected. This number was derived by performing a linear - // regression from the time the project was first available until November 30, 2006. - switch (type) { - case openfire: - count = 675774L; - break; - case spark: - count = 438159L; - break; - case smack: - count = 332007L; - break; - case xiff: - count = 4683L; - break; - default: - count = 0L; - } - - try { - con = connectionManager.getConnection(); - pstmt = con.prepareStatement(COUNT_TOTAL_DOWNLOADS_FOR_TYPE); - pstmt.setInt(1, type.getType()); - rs = pstmt.executeQuery(); - if (rs.next()) { - count += rs.getLong(1); - } - } - catch (Exception e) { - String name = null; - if (type != null) { - name = type.getName(); - } - Log.warn( "Error counting downloads for type " + name, e); - } - finally { - if (rs != null) { - try { rs.close(); } catch (Exception e) { Log.debug( "An exception occurred that can probably be ignored.", e); } - } - DbConnectionManager.close(pstmt, con); - } - counts.put(type.getName(), count); - } } } diff --git a/src/main/java/org/jivesoftware/webservices/RestClient.java b/src/main/java/org/jivesoftware/webservices/RestClient.java index 498d6d68..94024a09 100644 --- a/src/main/java/org/jivesoftware/webservices/RestClient.java +++ b/src/main/java/org/jivesoftware/webservices/RestClient.java @@ -12,6 +12,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.Map; public class RestClient { @@ -42,4 +43,37 @@ public JSONObject get(String url) { return result; } + + public JSONObject post(String url, Map headers, Map parameters) + { + JSONObject result = null; + + try (final CloseableHttpClient httpclient = CachingHttpClients.custom().setCacheConfig(cacheConfig).build()) + { + final ClassicRequestBuilder builder = ClassicRequestBuilder.post(url); + if (headers != null) { + for (Map.Entry header : headers.entrySet()) { + builder.addHeader(header.getKey(), header.getValue()); + } + } + if (parameters != null) { + for (Map.Entry entry : parameters.entrySet()) { + builder.addParameter(entry.getKey(), entry.getValue()); + } + } + final ClassicHttpRequest httpPost = builder.build(); + result = httpclient.execute(httpPost, response -> { + try { + return new JSONObject(EntityUtils.toString(response.getEntity())); + } catch (JSONException e) { + Log.warn("Invalid content while querying '{}'", url, e); + return null; + } + }); + } catch (IOException e) { + Log.warn("Fatal transport error while querying '{}'", url, e); + } + + return result; + } } diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index 7a9bb628..7897e76d 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -103,6 +103,11 @@ http://java.sun.com/xml/ns/j2ee/web-app_2_4.xsd" org.jivesoftware.site.DownloadStats 1 + + DiscourseAPI + org.jivesoftware.site.DiscourseAPI + 1 + DownloadServlet org.jivesoftware.site.DownloadServlet diff --git a/src/main/webapp/downloads/beta.jsp b/src/main/webapp/downloads/beta.jsp index e926b944..d40b7b2d 100644 --- a/src/main/webapp/downloads/beta.jsp +++ b/src/main/webapp/downloads/beta.jsp @@ -60,7 +60,7 @@
- +
diff --git a/src/main/webapp/downloads/index.jsp b/src/main/webapp/downloads/index.jsp index 4ec4cf8c..eef1cdc7 100644 --- a/src/main/webapp/downloads/index.jsp +++ b/src/main/webapp/downloads/index.jsp @@ -40,7 +40,7 @@
- +
diff --git a/src/main/webapp/downloads/nightly_openfire.jsp b/src/main/webapp/downloads/nightly_openfire.jsp index 01eea870..4c85a07a 100644 --- a/src/main/webapp/downloads/nightly_openfire.jsp +++ b/src/main/webapp/downloads/nightly_openfire.jsp @@ -164,7 +164,7 @@
- +
diff --git a/src/main/webapp/downloads/nightly_smack.jsp b/src/main/webapp/downloads/nightly_smack.jsp index e05aabad..239ffc56 100644 --- a/src/main/webapp/downloads/nightly_smack.jsp +++ b/src/main/webapp/downloads/nightly_smack.jsp @@ -118,7 +118,7 @@
- +
diff --git a/src/main/webapp/downloads/nightly_spark.jsp b/src/main/webapp/downloads/nightly_spark.jsp index 56621345..08d6319c 100644 --- a/src/main/webapp/downloads/nightly_spark.jsp +++ b/src/main/webapp/downloads/nightly_spark.jsp @@ -158,7 +158,7 @@
- +
diff --git a/src/main/webapp/downloads/nightly_xiff.jsp b/src/main/webapp/downloads/nightly_xiff.jsp index 55bab70a..118e8a01 100644 --- a/src/main/webapp/downloads/nightly_xiff.jsp +++ b/src/main/webapp/downloads/nightly_xiff.jsp @@ -118,7 +118,7 @@
- +
diff --git a/src/main/webapp/downloads/source.jsp b/src/main/webapp/downloads/source.jsp index 3eff5864..ba88ae7b 100644 --- a/src/main/webapp/downloads/source.jsp +++ b/src/main/webapp/downloads/source.jsp @@ -42,7 +42,7 @@
- +
diff --git a/src/main/webapp/fans/index.jsp b/src/main/webapp/fans/index.jsp index ba012779..51bb6107 100644 --- a/src/main/webapp/fans/index.jsp +++ b/src/main/webapp/fans/index.jsp @@ -187,7 +187,7 @@
- +
diff --git a/src/main/webapp/includes/sidebar_48hrsnapshot.jspf b/src/main/webapp/includes/sidebar_48hrsnapshot.jspf deleted file mode 100644 index f66c0211..00000000 --- a/src/main/webapp/includes/sidebar_48hrsnapshot.jspf +++ /dev/null @@ -1,36 +0,0 @@ -<%@ page import="java.text.NumberFormat, org.jivesoftware.site.DownloadStats"%> - -<%----%> - <%--
--%> - <%--
--%> - <%--
--%> - <%--
--%> - <%--
Downloads <%= NumberFormat.getNumberInstance().format(DownloadStats.getTotalDownloads()) %>
--%> - <%--<%–--%> - <%--// Jive V3 APIs do not provie user/forum/blog counts--%> - <%--UserService userService48 = serviceProvider.getUserService();--%> - <%--BlogService blogService48 = serviceProvider.getBlogService();--%> - <%--ForumService forumService48 = serviceProvider.getForumService();--%> - <%--WSResultFilter rf48 = new WSResultFilter();--%> - <%--rf48.setRecursive(true);--%> - <%--%>--%> - - <%----%> - <%--
Members--%> - <%--<%= userService48.getUserCount() %>--%> - <%--
--%> - <%--
Forum Posts--%> - <%--<%= forumService48.getMessageCountByCommunityIDAndFilter(1, rf48) %>--%> - <%--
--%> - <%--
Blog Entries--%> - <%--<%= blogService48.getBlogPostCount() %>--%> - <%--
--%> - <%--
--%> - <%--–%>--%> - <%--
--%> - <%--
--%> - <%--
--%> - <%----%> diff --git a/src/main/webapp/includes/sidebar_7daySnapshot.jspf b/src/main/webapp/includes/sidebar_7daySnapshot.jspf new file mode 100644 index 00000000..2a01952a --- /dev/null +++ b/src/main/webapp/includes/sidebar_7daySnapshot.jspf @@ -0,0 +1,37 @@ +<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core" %> +<%@ taglib prefix="fmt" uri="http://java.sun.com/jsp/jstl/fmt" %> +<%@ page import="org.jivesoftware.site.DownloadStats" %> +<%@ page import="org.jivesoftware.site.DiscourseAPI" %> +<% + request.setAttribute("downloadsLast7Days", DownloadStats.getTotalDownloadsLast7Days()); + request.setAttribute("activeMembers", DiscourseAPI.getActiveMembersLast7Days()); + request.setAttribute("newPosts", DiscourseAPI.getNewPostsLast7Days()); +%> +
+
+
+
+ +
+ Recent Downloads +
+
+ + + +
Members Active + +
+
+ +
New Forum Posts + +
+
+<%--
Blog Entries--%> +<%-- <%= blogService48.getBlogPostCount() %>--%> +<%--
--%> +
+
+
+
diff --git a/src/main/webapp/includes/sidebar_snapshot.jsp b/src/main/webapp/includes/sidebar_snapshot.jsp index b35f3181..b88ba9e0 100644 --- a/src/main/webapp/includes/sidebar_snapshot.jsp +++ b/src/main/webapp/includes/sidebar_snapshot.jsp @@ -1,15 +1,20 @@ -<%@ page import="org.jivesoftware.site.*, java.text.NumberFormat" %> +<%@ page import="org.jivesoftware.site.*" %> <%@ page import="org.slf4j.LoggerFactory" %> - +<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core" %> +<%@ taglib prefix="fmt" uri="http://java.sun.com/jsp/jstl/fmt" %> <% + String project = request.getParameter("project"); + request.setAttribute("version", Versions.getVersion(project)); long downloads = 0; try { downloads = DownloadStats.getDownloadsForType(DownloadServlet.DownloadInfo.valueOf(project)); } - catch (Exception e) { LoggerFactory.getLogger( this.getClass() ).debug( "An exception occurred that can probably be ignored.", e); } + catch (Exception e) { LoggerFactory.getLogger( "sidebar_snapshot.jsp" ).debug( "An exception occurred that can probably be ignored.", e); } + + request.setAttribute("downloads", downloads); // Grab the right license info String license = null; @@ -26,7 +31,7 @@ license = "Open Source Apache"; } else if (project.equals("tinder")) { - license = "Open Source GPL"; + license = "Open Source Apache"; } else if (project.equals("whack")) { license = "Open Source Apache"; @@ -37,14 +42,15 @@ else if (project.equals("xiff")) { license = "Open Source LGPL"; } + request.setAttribute("license", license); // Grab the right platform info String platform = null; if (project.equals("openfire")) { - platform = "Windows, Linux,
Unix, Mac OS X"; + platform = "Windows, Linux, Unix, Mac OS X"; } else if (project.equals("spark")) { - platform = "Windows, Linux,
Unix, Mac OS X"; + platform = "Windows, Linux, Unix, Mac OS X"; } else if (project.equals("sparkweb")) { platform = "Cross-Platform"; @@ -67,22 +73,22 @@ else if (project.equals("botz")) { platform = "Openfire"; } + request.setAttribute("platform", platform); %> -
-
Latest Build <%= Versions.getVersion(project) %>
- <% if (downloads != 0) { %> -
Downloads <%= NumberFormat.getNumberInstance().format(downloads) %>
- <% } %> - <% if (license != null) { %> -
License <%= license %>
- <% } %> - <% if (platform != null) { %> -
Platforms <%= platform %>
- <% } %> +
Latest Build
+ +
Downloads
+
+ +
License
+
+ +
Platforms
+
diff --git a/src/main/webapp/includes/sidebar_support.jspf b/src/main/webapp/includes/sidebar_support.jspf index 8df73387..477a94c6 100644 --- a/src/main/webapp/includes/sidebar_support.jspf +++ b/src/main/webapp/includes/sidebar_support.jspf @@ -1,4 +1,5 @@
+ diff --git a/src/main/webapp/index.jsp b/src/main/webapp/index.jsp index 45d78d91..49e4f6e7 100644 --- a/src/main/webapp/index.jsp +++ b/src/main/webapp/index.jsp @@ -132,8 +132,6 @@ - - diff --git a/src/main/webapp/news/index.jsp b/src/main/webapp/news/index.jsp index ac8393e6..97097d50 100644 --- a/src/main/webapp/news/index.jsp +++ b/src/main/webapp/news/index.jsp @@ -53,20 +53,6 @@
- - - - diff --git a/src/main/webapp/projects/index.jsp b/src/main/webapp/projects/index.jsp index fb2052e9..34f12f36 100644 --- a/src/main/webapp/projects/index.jsp +++ b/src/main/webapp/projects/index.jsp @@ -176,7 +176,7 @@
- +