diff --git a/docs/Query_Parameters/CachedMaps.md b/docs/Query_Parameters/CachedMaps.md new file mode 100644 index 00000000..03b8bd56 --- /dev/null +++ b/docs/Query_Parameters/CachedMaps.md @@ -0,0 +1,14 @@ +**CachedMaps:** + +`cachedMaps` is a CACHE user call to see the set of all cached map keys that the SuggestionEngine stores. + +You can see all cached reports by going to `/cachedMaps` REST endpoint. + +You can use the keys to query, for example, `/users?suggestion=` and get the full cached histogram report for that key. +Some example default keys are `numFilesUsers` and `diskspaceUsers`, which are cached reports of all users and their file counts and diskspace usage counts. + +Cached maps are updated every SuggestionEngine cycle visible on the Suggestion UI status page. + +Response code is 200 and a JSON representation of cached report keys available. + +Response code of 403 means you are not authorized to view this endpoint. \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hdfs/server/namenode/Constants.java b/src/main/java/org/apache/hadoop/hdfs/server/namenode/Constants.java index f62575dd..38cd2bde 100644 --- a/src/main/java/org/apache/hadoop/hdfs/server/namenode/Constants.java +++ b/src/main/java/org/apache/hadoop/hdfs/server/namenode/Constants.java @@ -282,7 +282,8 @@ enum Endpoint { metrics, setCachedQuery, getCachedQuery, - removeCachedQuery + removeCachedQuery, + cachedMaps } EnumSet UNSECURED_ENDPOINTS = @@ -312,7 +313,8 @@ enum Endpoint { Endpoint.fileAge, Endpoint.info, Endpoint.config, - Endpoint.getCachedQuery); + Endpoint.getCachedQuery, + Endpoint.cachedMaps); EnumSet READER_ENDPOINTS = EnumSet.of( diff --git a/src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/WebServerMain.java b/src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/WebServerMain.java index 49ba7e5b..bade4621 100644 --- a/src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/WebServerMain.java +++ b/src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/WebServerMain.java @@ -1586,6 +1586,15 @@ public void init( } }); + /* SUGGESTIONS endpoint is a cache-level endpoint meant to dump meta keys of cached analysis by NNA. */ + get( + "/cachedMaps", + (req, res) -> { + res.header("Access-Control-Allow-Origin", "*"); + res.header("Content-Type", "application/json"); + return Histograms.toJson(nameNodeLoader.getSuggestionsEngine().getCachedMapKeys()); + }); + /* DIRECTORIES endpoint is an reader-level endpoint meant to dump the cached directory analysis by NNA. */ get( "/directories", @@ -1699,8 +1708,32 @@ public void init( (req, res) -> { res.header("Access-Control-Allow-Origin", "*"); res.header("Content-Type", "application/json"); - String suggestion = req.queryMap("suggestion").value(); - return nameNodeLoader.getSuggestionsEngine().getUsersAsJson(suggestion); + final String suggestion = req.queryMap("suggestion").value(); + final Integer top = req.queryMap("top").integerValue(); + final String outputTypeStr = req.queryMap("histogramOutput").value(); + final String outputType = (outputTypeStr != null) ? outputTypeStr : "json"; + switch (outputType) { + case "json": + return nameNodeLoader.getSuggestionsEngine().getUsersAsJson(suggestion); + case "csv": + Map suggestionHistCsv = + nameNodeLoader.getSuggestionsEngine().getSuggestion(suggestion); + if (top != null && top > 0) { + suggestionHistCsv = Histograms.sliceToTop(suggestionHistCsv, top); + suggestionHistCsv = Histograms.sortByValue(suggestionHistCsv, false); + } + return Histograms.toCsv(suggestionHistCsv, null, true); + case "chart": + Map suggestionHistChart = + nameNodeLoader.getSuggestionsEngine().getSuggestion(suggestion); + if (top != null && top > 0) { + suggestionHistChart = Histograms.sliceToTop(suggestionHistChart, top); + suggestionHistChart = Histograms.sortByValue(suggestionHistChart, false); + } + return Histograms.toChartJsJson(suggestionHistChart, suggestion, "Username", "Count"); + default: + throw new IllegalArgumentException("Unknown histogram output type:" + outputType); + } }); /* TOP endpoint is an admin-level endpoint meant to dump the cached set of top issues by NNA. */ diff --git a/src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/web/NamenodeAnalyticsMethods.java b/src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/web/NamenodeAnalyticsMethods.java index 08256703..6abbec87 100644 --- a/src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/web/NamenodeAnalyticsMethods.java +++ b/src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/web/NamenodeAnalyticsMethods.java @@ -778,6 +778,28 @@ public Response suggestions() { } } + /** + * CACHEDMAPS endpoint is a cache-level endpoint meant to dump meta keys of cached analysis by + * NNA. + */ + @GET + @Path("/cachedMaps") + @Produces(MediaType.APPLICATION_JSON) + public Response cachedMaps() { + final NameNodeLoader nnLoader = (NameNodeLoader) context.getAttribute(NNA_NN_LOADER); + try { + before(); + return Response.ok( + Histograms.toJson(nnLoader.getSuggestionsEngine().getCachedMapKeys()), + MediaType.APPLICATION_JSON) + .build(); + } catch (Exception ex) { + return handleException(ex); + } finally { + after(); + } + } + /** * DIRECTORIES endpoint is an reader-level endpoint meant to dump the cached directory analysis by * NNA. diff --git a/src/main/java/org/apache/hadoop/hdfs/server/namenode/cache/SuggestionsEngine.java b/src/main/java/org/apache/hadoop/hdfs/server/namenode/cache/SuggestionsEngine.java index ad83de37..5bdd51b1 100644 --- a/src/main/java/org/apache/hadoop/hdfs/server/namenode/cache/SuggestionsEngine.java +++ b/src/main/java/org/apache/hadoop/hdfs/server/namenode/cache/SuggestionsEngine.java @@ -974,7 +974,7 @@ public String getUsersAsJson(String suggestion) { if (suggestion == null || suggestion.isEmpty()) { return Histograms.toJson(cachedUsers); } else { - Map userSuggestions = cachedMaps.get(suggestion); + Map userSuggestions = getSuggestion(suggestion); if (userSuggestions == null) { throw new IllegalArgumentException(suggestion + " is not a valid suggestion query."); } @@ -982,6 +982,19 @@ public String getUsersAsJson(String suggestion) { } } + /** + * Get all users' analysis of a particular suggestion cache as Histogram. + * + * @param suggestion the suggestion to fetch histogram for + * @return a histogram or null + */ + public Map getSuggestion(String suggestion) { + if (suggestion == null || suggestion.isEmpty()) { + return null; + } + return cachedMaps.get(suggestion); + } + /** * Get all users' analysis from cache as a JSON String. * @@ -1172,4 +1185,13 @@ public void start(ApplicationConfiguration conf) throws IOException { Collections.synchronizedMap(cacheManager.getCachedMapToMap("cachedMapQueries")); this.suggestionsReloadSleepMs = conf.getSuggestionsReloadSleepMs(); } + + /** + * Returns the set of all cached maps available for reporting. + * + * @return set of strings representing all cached maps available + */ + public Set getCachedMapKeys() { + return cachedMaps.keySet(); + } } diff --git a/src/main/resources/webapps/nna/cachedReport.html b/src/main/resources/webapps/nna/cachedReport.html new file mode 100644 index 00000000..9873ab39 --- /dev/null +++ b/src/main/resources/webapps/nna/cachedReport.html @@ -0,0 +1,459 @@ + + + + + + + + + + + + + + + + + NNA Report Histograms + + + + + + + + + + + + + +
+ + + +
+

Welcome to NNAnalytics

+
+

+ Get insight into your Hadoop clusters' data distribution.
+ Check "./cachedMaps" to see cached report options to compare / view. +

+ + +
+
+ + + + + +
+
+
+ + +
+
+ + + + + +
+
+ +
+ + + + + + + + + + + + + + diff --git a/src/main/resources/webapps/nna/css/nna.css b/src/main/resources/webapps/nna/css/nna.css index c00a2f22..141b3ae1 100644 --- a/src/main/resources/webapps/nna/css/nna.css +++ b/src/main/resources/webapps/nna/css/nna.css @@ -111,4 +111,46 @@ tbody.collapse.in { .list-group td:first-child { width: 75% !important; } +} + +.dropdown-submenu { + position:relative; +} +.dropdown-submenu>.dropdown-menu { + top:0; + left:100%; + margin-top:-6px; + margin-left:-1px; + -webkit-border-radius:0 6px 6px 6px; + -moz-border-radius:0 6px 6px 6px; + border-radius:0 6px 6px 6px; +} +.dropdown-submenu:hover>.dropdown-menu { + display:block; +} +.dropdown-submenu>a:after { + display:block; + content:" "; + float:right; + width:0; + height:0; + border-color:transparent; + border-style:solid; + border-width:5px 0 5px 5px; + border-left-color:#cccccc; + margin-top:5px; + margin-right:-10px; +} +.dropdown-submenu:hover>a:after { + border-left-color:#ffffff; +} +.dropdown-submenu.pull-left { + float:none; +} +.dropdown-submenu.pull-left>.dropdown-menu { + left:-100%; + margin-left:10px; + -webkit-border-radius:6px 0 6px 6px; + -moz-border-radius:6px 0 6px 6px; + border-radius:6px 0 6px 6px; } \ No newline at end of file diff --git a/src/main/resources/webapps/nna/history.html b/src/main/resources/webapps/nna/history.html index 4ed30a1a..fca33af5 100644 --- a/src/main/resources/webapps/nna/history.html +++ b/src/main/resources/webapps/nna/history.html @@ -600,7 +600,9 @@

Welcome to NNAnalytics

console.log(response); swal({ title: "", - text: response.responseText, + text: response.responseText + + "\nHistorical load on NNA is likely disabled." + + "\nPlease check ./info REST endpoint.", type: "error", showCancelButton: false, closeOnConfirm: true, diff --git a/src/main/resources/webapps/nna/js/nna.utility.js b/src/main/resources/webapps/nna/js/nna.utility.js index ca11a8da..e0663df5 100644 --- a/src/main/resources/webapps/nna/js/nna.utility.js +++ b/src/main/resources/webapps/nna/js/nna.utility.js @@ -443,9 +443,11 @@ function checkIfAdmin() { 400: function(xhr) { $("li.admin-menu").hide(); $("li.user-menu").hide(); + $("li.open-menu").show(); }, 200: function(xhr) { var isAdmin = xhr.includes("ADMIN"); + $("li.open-menu").hide(); if(isAdmin) { $("li.admin-menu").show(); } else { diff --git a/src/main/resources/webapps/nna/navbar.html b/src/main/resources/webapps/nna/navbar.html index 8bb99c72..fad9b571 100644 --- a/src/main/resources/webapps/nna/navbar.html +++ b/src/main/resources/webapps/nna/navbar.html @@ -56,130 +56,151 @@
  • Sign Out
    • -
  • Suggestions
    • -
  • History
    • - -
  • - - -
    • -
  • - - -
    • -
  • - - -
    • -
  • - - -
    • -
  • - +
    • +
  • Suggestions
    • +
  • History
  • - -