[#220] Split UI into Queries and Cached Reports and expose cached rep…

…orts

docs/Query_Parameters/CachedMaps.md

@@ -0,0 +1,14 @@
+**CachedMaps:**
+
+`cachedMaps` is a CACHE user call to see the set of all cached map keys that the SuggestionEngine stores.
+
+You can see all cached reports by going to `/cachedMaps` REST endpoint.
+
+You can use the keys to query, for example, `/users?suggestion=<key>` and get the full cached histogram report for that key.
+Some example default keys are `numFilesUsers` and `diskspaceUsers`, which are cached reports of all users and their file counts and diskspace usage counts.
+
+Cached maps are updated every SuggestionEngine cycle visible on the Suggestion UI status page.
+
+Response code is 200 and a JSON representation of cached report keys available.
+
+Response code of 403 means you are not authorized to view this endpoint.

src/main/java/org/apache/hadoop/hdfs/server/namenode/Constants.java

@@ -282,7 +282,8 @@ enum Endpoint {
     metrics,
     setCachedQuery,
     getCachedQuery,
-    removeCachedQuery
+    removeCachedQuery,
+    cachedMaps
   }
 
   EnumSet<Endpoint> UNSECURED_ENDPOINTS =
@@ -312,7 +313,8 @@ enum Endpoint {
           Endpoint.fileAge,
           Endpoint.info,
           Endpoint.config,
-          Endpoint.getCachedQuery);
+          Endpoint.getCachedQuery,
+          Endpoint.cachedMaps);
 
   EnumSet<Endpoint> READER_ENDPOINTS =
       EnumSet.of(

src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/WebServerMain.java

@@ -1586,6 +1586,15 @@ public void init(
           }
         });
 
+    /* SUGGESTIONS endpoint is a cache-level endpoint meant to dump meta keys of cached analysis by NNA. */
+    get(
+        "/cachedMaps",
+        (req, res) -> {
+          res.header("Access-Control-Allow-Origin", "*");
+          res.header("Content-Type", "application/json");
+          return Histograms.toJson(nameNodeLoader.getSuggestionsEngine().getCachedMapKeys());
+        });
+
     /* DIRECTORIES endpoint is an reader-level endpoint meant to dump the cached directory analysis by NNA. */
     get(
         "/directories",
@@ -1699,8 +1708,32 @@ public void init(
         (req, res) -> {
           res.header("Access-Control-Allow-Origin", "*");
           res.header("Content-Type", "application/json");
-          String suggestion = req.queryMap("suggestion").value();
-          return nameNodeLoader.getSuggestionsEngine().getUsersAsJson(suggestion);
+          final String suggestion = req.queryMap("suggestion").value();
+          final Integer top = req.queryMap("top").integerValue();
+          final String outputTypeStr = req.queryMap("histogramOutput").value();
+          final String outputType = (outputTypeStr != null) ? outputTypeStr : "json";
+          switch (outputType) {
+            case "json":
+              return nameNodeLoader.getSuggestionsEngine().getUsersAsJson(suggestion);
+            case "csv":
+              Map<String, Long> suggestionHistCsv =
+                  nameNodeLoader.getSuggestionsEngine().getSuggestion(suggestion);
+              if (top != null && top > 0) {
+                suggestionHistCsv = Histograms.sliceToTop(suggestionHistCsv, top);
+                suggestionHistCsv = Histograms.sortByValue(suggestionHistCsv, false);
+              }
+              return Histograms.toCsv(suggestionHistCsv, null, true);
+            case "chart":
+              Map<String, Long> suggestionHistChart =
+                  nameNodeLoader.getSuggestionsEngine().getSuggestion(suggestion);
+              if (top != null && top > 0) {
+                suggestionHistChart = Histograms.sliceToTop(suggestionHistChart, top);
+                suggestionHistChart = Histograms.sortByValue(suggestionHistChart, false);
+              }
+              return Histograms.toChartJsJson(suggestionHistChart, suggestion, "Username", "Count");
+            default:
+              throw new IllegalArgumentException("Unknown histogram output type:" + outputType);
+          }
         });
 
     /* TOP endpoint is an admin-level endpoint meant to dump the cached set of top issues by NNA. */

src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/web/NamenodeAnalyticsMethods.java

@@ -778,6 +778,28 @@ public Response suggestions() {
     }
   }
 
+  /**
+   * CACHEDMAPS endpoint is a cache-level endpoint meant to dump meta keys of cached analysis by
+   * NNA.
+   */
+  @GET
+  @Path("/cachedMaps")
+  @Produces(MediaType.APPLICATION_JSON)
+  public Response cachedMaps() {
+    final NameNodeLoader nnLoader = (NameNodeLoader) context.getAttribute(NNA_NN_LOADER);
+    try {
+      before();
+      return Response.ok(
+              Histograms.toJson(nnLoader.getSuggestionsEngine().getCachedMapKeys()),
+              MediaType.APPLICATION_JSON)
+          .build();
+    } catch (Exception ex) {
+      return handleException(ex);
+    } finally {
+      after();
+    }
+  }
+
   /**
    * DIRECTORIES endpoint is an reader-level endpoint meant to dump the cached directory analysis by
    * NNA.

src/main/java/org/apache/hadoop/hdfs/server/namenode/cache/SuggestionsEngine.java

@@ -974,14 +974,27 @@ public String getUsersAsJson(String suggestion) {
     if (suggestion == null || suggestion.isEmpty()) {
       return Histograms.toJson(cachedUsers);
     } else {
-      Map<String, Long> userSuggestions = cachedMaps.get(suggestion);
+      Map<String, Long> userSuggestions = getSuggestion(suggestion);
       if (userSuggestions == null) {
         throw new IllegalArgumentException(suggestion + " is not a valid suggestion query.");
       }
       return Histograms.toJson(userSuggestions);
     }
   }
 
+  /**
+   * Get all users' analysis of a particular suggestion cache as Histogram.
+   *
+   * @param suggestion the suggestion to fetch histogram for
+   * @return a histogram or null
+   */
+  public Map<String, Long> getSuggestion(String suggestion) {
+    if (suggestion == null || suggestion.isEmpty()) {
+      return null;
+    }
+    return cachedMaps.get(suggestion);
+  }
+
   /**
    * Get all users' analysis from cache as a JSON String.
    *
@@ -1172,4 +1185,13 @@ public void start(ApplicationConfiguration conf) throws IOException {
         Collections.synchronizedMap(cacheManager.getCachedMapToMap("cachedMapQueries"));
     this.suggestionsReloadSleepMs = conf.getSuggestionsReloadSleepMs();
   }
+
+  /**
+   * Returns the set of all cached maps available for reporting.
+   *
+   * @return set of strings representing all cached maps available
+   */
+  public Set<String> getCachedMapKeys() {
+    return cachedMaps.keySet();
+  }
 }