[#223] Externalize web resources

build.gradle

@@ -228,5 +228,10 @@ ospackage {
         fileType CONFIG | NOREPLACE
         into 'config'
     }
+    from('src/main/resources/webapps') {
+        fileType CONFIG | NOREPLACE
+        into 'webapps'
+        fileMode = 0755
+    }
 
 }

docs/Getting_Started/How_To_Configure.md

@@ -23,6 +23,8 @@ For example:
 
 Definitions of configuration NNA-specific properties:
 
+* `nna.base.dir=<path>` - Default is `/usr/local/nn-analytics`. Change this if you plan to install / move NNA into a different directory.
+* `nna.web.base.dir=<path>` - Default is `/usr/local/nn-analytics/webapps/nna`. Change this is you plan to load NNA web resources from a different directory.
 * `nna.port=<integer>` - Default is 8080. Represents the main web UI port.
 * `nna.historical=<true | false>` - Default is false. True enables a locally embedded HSQL DB to trend data. Not recommended in production.
 * `nna.support.bootstrap.overrides=<true | false>` - Default is true. True will override certain hdfs-site.xml configurations to prevent NNA from communicating with the active cluster. False means it will use configurations as-is. Recommended true in production.

src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/ApplicationConfiguration.java

@@ -49,6 +49,7 @@ public class ApplicationConfiguration {
   private static final String LDAP_RESPONSE_TIMEOUT_DEFAULT = "1000";
   private static final String NNA_SUGGESTIONS_RELOAD_TIMEOUT_DEFAULT = "900000";
   private static final String NNA_BASE_DIR_DEFAULT = "/usr/local/nn-analytics";
+  private static final String NNA_WEB_BASE_DIR_DEFAULT = NNA_BASE_DIR_DEFAULT + "/webapps/nna";
   private static final String NNA_SUPPORT_BOOTSTRAP_OVERRIDES = "true";
   private static final String NNA_QUERY_ENGINE_DEFAULT =
       JavaStreamQueryEngine.class.getCanonicalName();
@@ -86,6 +87,10 @@ public String getBaseDir() {
     return properties.getProperty("nna.base.dir", NNA_BASE_DIR_DEFAULT);
   }
 
+  public String getWebBaseDir() {
+    return properties.getProperty("nna.web.base.dir", NNA_WEB_BASE_DIR_DEFAULT);
+  }
+
   public boolean getHistoricalEnabled() {
     return Boolean.parseBoolean(properties.getProperty("nna.historical", NNA_HISTORICAL_DEFAULT));
   }

src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/NameNodeAnalyticsHttpServer.java

@@ -180,6 +180,7 @@ public void start() throws IOException {
     httpServer = builder.build();
 
     // Main load of NNA classes and API into NNA Http Server.
+    httpServer.getWebAppContext().setWar(nnaConf.getWebBaseDir());
     httpServer.getWebAppContext().setAttribute(NNA_NN_LOADER, nnLoader);
     httpServer.getWebAppContext().setAttribute(NNA_SECURITY_CONTEXT, secContext);
     httpServer.getWebAppContext().setAttribute(NNA_USAGE_METRICS, usageMetrics);

src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/WebServerMain.java

@@ -294,7 +294,7 @@ public void init(
     }
 
     /* This is the call to load everything under ./resources/public as HTML resources. */
-    Spark.staticFileLocation("/webapps/nna");
+    Spark.externalStaticFileLocation(conf.getWebBaseDir());
 
     /* LOGIN is used to log into authenticated web sessions. */
     post(

src/main/java/org/apache/hadoop/hdfs/server/namenode/cache/CachedDirectories.java

@@ -70,17 +70,19 @@ public void analyze(
     long start = System.currentTimeMillis();
 
     /* Make an in-mem copy of the cachedDirs so we can parallelize the stream. */
-    HashSet<String> inMemCachedDirsCopy = new HashSet<>(cachedDirs);
-    Map<String, ContentSummary> contentSummaries =
-        inMemCachedDirsCopy
-            .parallelStream()
-            .collect(Collectors.toMap(Function.identity(), nnLoader::getContentSummary));
-    for (Entry<String, ContentSummary> entry : contentSummaries.entrySet()) {
-      if (entry.getKey() == null || entry.getValue() == null) {
-        continue;
+    if (cachedDirs.size() > 0) {
+      HashSet<String> inMemCachedDirsCopy = new HashSet<>(cachedDirs);
+      Map<String, ContentSummary> contentSummaries =
+          inMemCachedDirsCopy
+              .parallelStream()
+              .collect(Collectors.toMap(Function.identity(), nnLoader::getContentSummary));
+      for (Entry<String, ContentSummary> entry : contentSummaries.entrySet()) {
+        if (entry.getKey() == null || entry.getValue() == null) {
+          continue;
+        }
+        countMap.put(entry.getKey(), entry.getValue().getFileCount());
+        diskspaceMap.put(entry.getKey(), entry.getValue().getSpaceConsumed());
       }
-      countMap.put(entry.getKey(), entry.getValue().getFileCount());
-      diskspaceMap.put(entry.getKey(), entry.getValue().getSpaceConsumed());
     }
 
     long end = System.currentTimeMillis();

src/main/resources/config/application.properties

@@ -20,6 +20,9 @@
 # Installation base dir.
 # nna.base.dir=/usr/local/nn-analytics
 
+# Web resources base dir.
+# nna.web.base.dir=/usr/local/nn-analytics/webapps/nna
+
 # REST API Port for NNA.
 nna.port=8080
 

src/main/resources/env/nna_env

@@ -21,7 +21,7 @@
 
 export WORKING_DIR=/usr/local/nn-analytics
 export JAVA_HOME=/usr/java/latest
-export JAVA_CLASSPATH=${WORKING_DIR}/config:${WORKING_DIR}/lib/*
+export JAVA_CLASSPATH=${WORKING_DIR}/config:${WORKING_DIR}/lib/*:${WORKING_DIR}/webapps
 
 export NNA_JVM_OPTIONS="-server -XX:ParallelGCThreads=8 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:NewSize=25600m -XX:MaxNewSize=25600m \
 -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps \

src/test/3.0.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAAuthorization.java

@@ -42,6 +42,7 @@ public static void beforeClass() throws Exception {
     conf.set("ldap.enable", "false");
     conf.set("authorization.enable", "true");
     conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
+    conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
     nna.init(conf, gset);
     hostPort = new HttpHost("localhost", 4567);
   }

src/test/3.0.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAOperations.java

@@ -44,6 +44,7 @@ public static void beforeClass() throws Exception {
     conf.set("authorization.enable", "false");
     conf.set("nna.historical", "false");
     conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
+    conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
     conf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName());
     nna.init(conf, gset);
     hostPort = new HttpHost("localhost", 4567);

src/test/3.0.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAWithStreamEngine.java

@@ -52,6 +52,7 @@ public static void beforeClass() throws Exception {
     conf.set("authorization.enable", "false");
     conf.set("nna.historical", "false");
     conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
+    conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
     conf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName());
     nna.init(conf, gset);
     hostPort = new HttpHost("localhost", 4567);

src/test/3.0.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestMiniClusterHadoopNNAWithStreamEngine.java

@@ -80,6 +80,7 @@ public static void beforeClass() throws Exception {
     nnaConf.set("authorization.enable", "false");
     nnaConf.set("nna.historical", "true");
     nnaConf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
+    nnaConf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
     nnaConf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName());
     nna.init(nnaConf, null, CONF);
     hostPort = new HttpHost("localhost", 4567);

src/test/3.1.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAAuthorization.java

@@ -42,6 +42,7 @@ public static void beforeClass() throws Exception {
     conf.set("ldap.enable", "false");
     conf.set("authorization.enable", "true");
     conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
+    conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
     nna.init(conf, gset);
     hostPort = new HttpHost("localhost", 4567);
   }

src/test/3.1.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAOperations.java

@@ -44,6 +44,7 @@ public static void beforeClass() throws Exception {
     conf.set("authorization.enable", "false");
     conf.set("nna.historical", "false");
     conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
+    conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
     conf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName());
     nna.init(conf, gset);
     hostPort = new HttpHost("localhost", 4567);

src/test/3.1.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAWithStreamEngine.java

@@ -52,6 +52,7 @@ public static void beforeClass() throws Exception {
     conf.set("authorization.enable", "false");
     conf.set("nna.historical", "false");
     conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
+    conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
     conf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName());
     nna.init(conf, gset);
     hostPort = new HttpHost("localhost", 4567);

src/test/3.1.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestMiniClusterHadoopNNAWithStreamEngine.java

@@ -80,6 +80,7 @@ public static void beforeClass() throws Exception {
     nnaConf.set("authorization.enable", "false");
     nnaConf.set("nna.historical", "true");
     nnaConf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
+    nnaConf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
     nnaConf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName());
     nna.init(nnaConf, null, CONF);
     hostPort = new HttpHost("localhost", 4567);

src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestLdapAuth.java

@@ -69,6 +69,7 @@ public static void beforeClass() throws Exception {
     conf.set("nna.historical", "false");
     conf.set("nna.localonly.users", "hdfs:hdfs,hdfsW:hdfsW,hdfsR:hdfsR,testEmpty:");
     conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
+    conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
     nna.init(conf, gset);
     hostPort = new HttpHost("localhost", 4567);
   }

src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestNNAWithStreamEngine.java

@@ -52,6 +52,7 @@ public static void beforeClass() throws Exception {
     conf.set("authorization.enable", "false");
     conf.set("nna.historical", "false");
     conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
+    conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
     conf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName());
     nna.init(conf, gset);
     hostPort = new HttpHost("localhost", 4567);

src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestNoHistorical.java

@@ -57,6 +57,7 @@ public static void beforeClass() throws Exception {
     conf.set("authorization.enable", "false");
     conf.set("nna.historical", "false");
     conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
+    conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
     nna.init(conf, gset);
     hostPort = new HttpHost("localhost", 4567);
   }

src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWebServerMainAuthorization.java

@@ -42,6 +42,7 @@ public static void beforeClass() throws Exception {
     conf.set("ldap.enable", "false");
     conf.set("authorization.enable", "true");
     conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
+    conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
     nna.init(conf, gset);
     hostPort = new HttpHost("localhost", 4567);
   }

src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWebServerMainOperations.java

@@ -43,6 +43,7 @@ public static void beforeClass() throws Exception {
     conf.set("authorization.enable", "false");
     conf.set("nna.historical", "false");
     conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
+    conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
     nna.init(conf, gset);
     hostPort = new HttpHost("localhost", 4567);
   }

src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWithMiniClusterWithStreamEngine.java

@@ -80,6 +80,7 @@ public static void beforeClass() throws Exception {
     nnaConf.set("authorization.enable", "false");
     nnaConf.set("nna.historical", "true");
     nnaConf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
+    nnaConf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
     nnaConf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName());
     nna.init(nnaConf, null, CONF);
     hostPort = new HttpHost("localhost", 4567);

src/test/resources/application.properties

@@ -20,6 +20,9 @@
 # Installation base dir.
 # nna.base.dir=/usr/local/nn-analytics
 
+# Web resources base dir.
+# nna.web.base.dir=webapps/nna
+
 # REST API Port for NNA.
 nna.port=4567