diff --git a/build.gradle b/build.gradle index 0e253fb8..64d96d34 100644 --- a/build.gradle +++ b/build.gradle @@ -228,5 +228,10 @@ ospackage { fileType CONFIG | NOREPLACE into 'config' } + from('src/main/resources/webapps') { + fileType CONFIG | NOREPLACE + into 'webapps' + fileMode = 0755 + } } diff --git a/docs/Getting_Started/How_To_Configure.md b/docs/Getting_Started/How_To_Configure.md index bc81d518..dab98b41 100644 --- a/docs/Getting_Started/How_To_Configure.md +++ b/docs/Getting_Started/How_To_Configure.md @@ -23,6 +23,8 @@ For example: Definitions of configuration NNA-specific properties: +* `nna.base.dir=` - Default is `/usr/local/nn-analytics`. Change this if you plan to install / move NNA into a different directory. +* `nna.web.base.dir=` - Default is `/usr/local/nn-analytics/webapps/nna`. Change this is you plan to load NNA web resources from a different directory. * `nna.port=` - Default is 8080. Represents the main web UI port. * `nna.historical=` - Default is false. True enables a locally embedded HSQL DB to trend data. Not recommended in production. * `nna.support.bootstrap.overrides=` - Default is true. True will override certain hdfs-site.xml configurations to prevent NNA from communicating with the active cluster. False means it will use configurations as-is. Recommended true in production. diff --git a/src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/ApplicationConfiguration.java b/src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/ApplicationConfiguration.java index 179293fa..03cf0f58 100644 --- a/src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/ApplicationConfiguration.java +++ b/src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/ApplicationConfiguration.java @@ -49,6 +49,7 @@ public class ApplicationConfiguration { private static final String LDAP_RESPONSE_TIMEOUT_DEFAULT = "1000"; private static final String NNA_SUGGESTIONS_RELOAD_TIMEOUT_DEFAULT = "900000"; private static final String NNA_BASE_DIR_DEFAULT = "/usr/local/nn-analytics"; + private static final String NNA_WEB_BASE_DIR_DEFAULT = NNA_BASE_DIR_DEFAULT + "/webapps/nna"; private static final String NNA_SUPPORT_BOOTSTRAP_OVERRIDES = "true"; private static final String NNA_QUERY_ENGINE_DEFAULT = JavaStreamQueryEngine.class.getCanonicalName(); @@ -86,6 +87,10 @@ public String getBaseDir() { return properties.getProperty("nna.base.dir", NNA_BASE_DIR_DEFAULT); } + public String getWebBaseDir() { + return properties.getProperty("nna.web.base.dir", NNA_WEB_BASE_DIR_DEFAULT); + } + public boolean getHistoricalEnabled() { return Boolean.parseBoolean(properties.getProperty("nna.historical", NNA_HISTORICAL_DEFAULT)); } diff --git a/src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/NameNodeAnalyticsHttpServer.java b/src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/NameNodeAnalyticsHttpServer.java index 30a72966..9188e212 100644 --- a/src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/NameNodeAnalyticsHttpServer.java +++ b/src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/NameNodeAnalyticsHttpServer.java @@ -180,6 +180,7 @@ public void start() throws IOException { httpServer = builder.build(); // Main load of NNA classes and API into NNA Http Server. + httpServer.getWebAppContext().setWar(nnaConf.getWebBaseDir()); httpServer.getWebAppContext().setAttribute(NNA_NN_LOADER, nnLoader); httpServer.getWebAppContext().setAttribute(NNA_SECURITY_CONTEXT, secContext); httpServer.getWebAppContext().setAttribute(NNA_USAGE_METRICS, usageMetrics); diff --git a/src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/WebServerMain.java b/src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/WebServerMain.java index e751bc55..36c0b4f6 100644 --- a/src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/WebServerMain.java +++ b/src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/WebServerMain.java @@ -294,7 +294,7 @@ public void init( } /* This is the call to load everything under ./resources/public as HTML resources. */ - Spark.staticFileLocation("/webapps/nna"); + Spark.externalStaticFileLocation(conf.getWebBaseDir()); /* LOGIN is used to log into authenticated web sessions. */ post( diff --git a/src/main/java/org/apache/hadoop/hdfs/server/namenode/cache/CachedDirectories.java b/src/main/java/org/apache/hadoop/hdfs/server/namenode/cache/CachedDirectories.java index 08d1e4f1..f0ca13e2 100644 --- a/src/main/java/org/apache/hadoop/hdfs/server/namenode/cache/CachedDirectories.java +++ b/src/main/java/org/apache/hadoop/hdfs/server/namenode/cache/CachedDirectories.java @@ -70,17 +70,19 @@ public void analyze( long start = System.currentTimeMillis(); /* Make an in-mem copy of the cachedDirs so we can parallelize the stream. */ - HashSet inMemCachedDirsCopy = new HashSet<>(cachedDirs); - Map contentSummaries = - inMemCachedDirsCopy - .parallelStream() - .collect(Collectors.toMap(Function.identity(), nnLoader::getContentSummary)); - for (Entry entry : contentSummaries.entrySet()) { - if (entry.getKey() == null || entry.getValue() == null) { - continue; + if (cachedDirs.size() > 0) { + HashSet inMemCachedDirsCopy = new HashSet<>(cachedDirs); + Map contentSummaries = + inMemCachedDirsCopy + .parallelStream() + .collect(Collectors.toMap(Function.identity(), nnLoader::getContentSummary)); + for (Entry entry : contentSummaries.entrySet()) { + if (entry.getKey() == null || entry.getValue() == null) { + continue; + } + countMap.put(entry.getKey(), entry.getValue().getFileCount()); + diskspaceMap.put(entry.getKey(), entry.getValue().getSpaceConsumed()); } - countMap.put(entry.getKey(), entry.getValue().getFileCount()); - diskspaceMap.put(entry.getKey(), entry.getValue().getSpaceConsumed()); } long end = System.currentTimeMillis(); diff --git a/src/main/resources/config/application.properties b/src/main/resources/config/application.properties index d348b621..5736bea1 100644 --- a/src/main/resources/config/application.properties +++ b/src/main/resources/config/application.properties @@ -20,6 +20,9 @@ # Installation base dir. # nna.base.dir=/usr/local/nn-analytics +# Web resources base dir. +# nna.web.base.dir=/usr/local/nn-analytics/webapps/nna + # REST API Port for NNA. nna.port=8080 diff --git a/src/main/resources/env/nna_env b/src/main/resources/env/nna_env index e560ea56..f531e44c 100644 --- a/src/main/resources/env/nna_env +++ b/src/main/resources/env/nna_env @@ -21,7 +21,7 @@ export WORKING_DIR=/usr/local/nn-analytics export JAVA_HOME=/usr/java/latest -export JAVA_CLASSPATH=${WORKING_DIR}/config:${WORKING_DIR}/lib/* +export JAVA_CLASSPATH=${WORKING_DIR}/config:${WORKING_DIR}/lib/*:${WORKING_DIR}/webapps export NNA_JVM_OPTIONS="-server -XX:ParallelGCThreads=8 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:NewSize=25600m -XX:MaxNewSize=25600m \ -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps \ diff --git a/src/test/3.0.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAAuthorization.java b/src/test/3.0.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAAuthorization.java index 2228917e..afb171f3 100644 --- a/src/test/3.0.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAAuthorization.java +++ b/src/test/3.0.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAAuthorization.java @@ -42,6 +42,7 @@ public static void beforeClass() throws Exception { conf.set("ldap.enable", "false"); conf.set("authorization.enable", "true"); conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory()); + conf.set("nna.web.base.dir", "src/main/resources/webapps/nna"); nna.init(conf, gset); hostPort = new HttpHost("localhost", 4567); } diff --git a/src/test/3.0.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAOperations.java b/src/test/3.0.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAOperations.java index 2eab1888..c3579dd0 100644 --- a/src/test/3.0.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAOperations.java +++ b/src/test/3.0.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAOperations.java @@ -44,6 +44,7 @@ public static void beforeClass() throws Exception { conf.set("authorization.enable", "false"); conf.set("nna.historical", "false"); conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory()); + conf.set("nna.web.base.dir", "src/main/resources/webapps/nna"); conf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName()); nna.init(conf, gset); hostPort = new HttpHost("localhost", 4567); diff --git a/src/test/3.0.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAWithStreamEngine.java b/src/test/3.0.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAWithStreamEngine.java index ef21ade1..60e9d4fb 100644 --- a/src/test/3.0.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAWithStreamEngine.java +++ b/src/test/3.0.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAWithStreamEngine.java @@ -52,6 +52,7 @@ public static void beforeClass() throws Exception { conf.set("authorization.enable", "false"); conf.set("nna.historical", "false"); conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory()); + conf.set("nna.web.base.dir", "src/main/resources/webapps/nna"); conf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName()); nna.init(conf, gset); hostPort = new HttpHost("localhost", 4567); diff --git a/src/test/3.0.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestMiniClusterHadoopNNAWithStreamEngine.java b/src/test/3.0.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestMiniClusterHadoopNNAWithStreamEngine.java index facd8ac0..c590ad79 100644 --- a/src/test/3.0.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestMiniClusterHadoopNNAWithStreamEngine.java +++ b/src/test/3.0.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestMiniClusterHadoopNNAWithStreamEngine.java @@ -80,6 +80,7 @@ public static void beforeClass() throws Exception { nnaConf.set("authorization.enable", "false"); nnaConf.set("nna.historical", "true"); nnaConf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory()); + nnaConf.set("nna.web.base.dir", "src/main/resources/webapps/nna"); nnaConf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName()); nna.init(nnaConf, null, CONF); hostPort = new HttpHost("localhost", 4567); diff --git a/src/test/3.1.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAAuthorization.java b/src/test/3.1.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAAuthorization.java index 2228917e..afb171f3 100644 --- a/src/test/3.1.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAAuthorization.java +++ b/src/test/3.1.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAAuthorization.java @@ -42,6 +42,7 @@ public static void beforeClass() throws Exception { conf.set("ldap.enable", "false"); conf.set("authorization.enable", "true"); conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory()); + conf.set("nna.web.base.dir", "src/main/resources/webapps/nna"); nna.init(conf, gset); hostPort = new HttpHost("localhost", 4567); } diff --git a/src/test/3.1.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAOperations.java b/src/test/3.1.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAOperations.java index 2eab1888..c3579dd0 100644 --- a/src/test/3.1.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAOperations.java +++ b/src/test/3.1.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAOperations.java @@ -44,6 +44,7 @@ public static void beforeClass() throws Exception { conf.set("authorization.enable", "false"); conf.set("nna.historical", "false"); conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory()); + conf.set("nna.web.base.dir", "src/main/resources/webapps/nna"); conf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName()); nna.init(conf, gset); hostPort = new HttpHost("localhost", 4567); diff --git a/src/test/3.1.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAWithStreamEngine.java b/src/test/3.1.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAWithStreamEngine.java index ef21ade1..60e9d4fb 100644 --- a/src/test/3.1.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAWithStreamEngine.java +++ b/src/test/3.1.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestHadoopNNAWithStreamEngine.java @@ -52,6 +52,7 @@ public static void beforeClass() throws Exception { conf.set("authorization.enable", "false"); conf.set("nna.historical", "false"); conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory()); + conf.set("nna.web.base.dir", "src/main/resources/webapps/nna"); conf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName()); nna.init(conf, gset); hostPort = new HttpHost("localhost", 4567); diff --git a/src/test/3.1.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestMiniClusterHadoopNNAWithStreamEngine.java b/src/test/3.1.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestMiniClusterHadoopNNAWithStreamEngine.java index facd8ac0..c590ad79 100644 --- a/src/test/3.1.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestMiniClusterHadoopNNAWithStreamEngine.java +++ b/src/test/3.1.0/org/apache/hadoop/hdfs/server/namenode/analytics/TestMiniClusterHadoopNNAWithStreamEngine.java @@ -80,6 +80,7 @@ public static void beforeClass() throws Exception { nnaConf.set("authorization.enable", "false"); nnaConf.set("nna.historical", "true"); nnaConf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory()); + nnaConf.set("nna.web.base.dir", "src/main/resources/webapps/nna"); nnaConf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName()); nna.init(nnaConf, null, CONF); hostPort = new HttpHost("localhost", 4567); diff --git a/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestLdapAuth.java b/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestLdapAuth.java index 5fdd4579..a50875cb 100644 --- a/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestLdapAuth.java +++ b/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestLdapAuth.java @@ -69,6 +69,7 @@ public static void beforeClass() throws Exception { conf.set("nna.historical", "false"); conf.set("nna.localonly.users", "hdfs:hdfs,hdfsW:hdfsW,hdfsR:hdfsR,testEmpty:"); conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory()); + conf.set("nna.web.base.dir", "src/main/resources/webapps/nna"); nna.init(conf, gset); hostPort = new HttpHost("localhost", 4567); } diff --git a/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestNNAWithStreamEngine.java b/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestNNAWithStreamEngine.java index 1a08e4ac..71740547 100644 --- a/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestNNAWithStreamEngine.java +++ b/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestNNAWithStreamEngine.java @@ -52,6 +52,7 @@ public static void beforeClass() throws Exception { conf.set("authorization.enable", "false"); conf.set("nna.historical", "false"); conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory()); + conf.set("nna.web.base.dir", "src/main/resources/webapps/nna"); conf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName()); nna.init(conf, gset); hostPort = new HttpHost("localhost", 4567); diff --git a/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestNoHistorical.java b/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestNoHistorical.java index db77a9da..bc9780d8 100644 --- a/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestNoHistorical.java +++ b/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestNoHistorical.java @@ -57,6 +57,7 @@ public static void beforeClass() throws Exception { conf.set("authorization.enable", "false"); conf.set("nna.historical", "false"); conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory()); + conf.set("nna.web.base.dir", "src/main/resources/webapps/nna"); nna.init(conf, gset); hostPort = new HttpHost("localhost", 4567); } diff --git a/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWebServerMainAuthorization.java b/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWebServerMainAuthorization.java index 4f502632..728fd653 100644 --- a/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWebServerMainAuthorization.java +++ b/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWebServerMainAuthorization.java @@ -42,6 +42,7 @@ public static void beforeClass() throws Exception { conf.set("ldap.enable", "false"); conf.set("authorization.enable", "true"); conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory()); + conf.set("nna.web.base.dir", "src/main/resources/webapps/nna"); nna.init(conf, gset); hostPort = new HttpHost("localhost", 4567); } diff --git a/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWebServerMainOperations.java b/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWebServerMainOperations.java index 89a47b13..f6ac6304 100644 --- a/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWebServerMainOperations.java +++ b/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWebServerMainOperations.java @@ -43,6 +43,7 @@ public static void beforeClass() throws Exception { conf.set("authorization.enable", "false"); conf.set("nna.historical", "false"); conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory()); + conf.set("nna.web.base.dir", "src/main/resources/webapps/nna"); nna.init(conf, gset); hostPort = new HttpHost("localhost", 4567); } diff --git a/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWithMiniClusterWithStreamEngine.java b/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWithMiniClusterWithStreamEngine.java index 37873d27..b223c32a 100644 --- a/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWithMiniClusterWithStreamEngine.java +++ b/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWithMiniClusterWithStreamEngine.java @@ -80,6 +80,7 @@ public static void beforeClass() throws Exception { nnaConf.set("authorization.enable", "false"); nnaConf.set("nna.historical", "true"); nnaConf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory()); + nnaConf.set("nna.web.base.dir", "src/main/resources/webapps/nna"); nnaConf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName()); nna.init(nnaConf, null, CONF); hostPort = new HttpHost("localhost", 4567); diff --git a/src/test/resources/application.properties b/src/test/resources/application.properties index 564c5c95..20b5c600 100644 --- a/src/test/resources/application.properties +++ b/src/test/resources/application.properties @@ -20,6 +20,9 @@ # Installation base dir. # nna.base.dir=/usr/local/nn-analytics +# Web resources base dir. +# nna.web.base.dir=webapps/nna + # REST API Port for NNA. nna.port=4567