From ceca84be071eff86639f91ebe2d039fc58754ff6 Mon Sep 17 00:00:00 2001 From: Plamen Jeliazkov Date: Wed, 5 Jun 2019 13:50:44 -0700 Subject: [PATCH] [#243] File type histogram not detecting app logs --- .../server/namenode/queries/FileTypeHistogram.java | 4 ++++ .../analytics/TestWithMiniClusterBase.java | 14 ++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/apache/hadoop/hdfs/server/namenode/queries/FileTypeHistogram.java b/src/main/java/org/apache/hadoop/hdfs/server/namenode/queries/FileTypeHistogram.java index 3617d898..46c478d7 100644 --- a/src/main/java/org/apache/hadoop/hdfs/server/namenode/queries/FileTypeHistogram.java +++ b/src/main/java/org/apache/hadoop/hdfs/server/namenode/queries/FileTypeHistogram.java @@ -188,6 +188,10 @@ public static String determineType(String name) { if (extIndex != -1 && (key = suffixExtMap.get(name.substring(extIndex))) != null) { return key; } + extIndex = name.lastIndexOf("_"); + if (extIndex != -1 && (key = suffixExtMap.get(name.substring(extIndex))) != null) { + return key; + } } return Types.UNKNOWN.name(); } diff --git a/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWithMiniClusterBase.java b/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWithMiniClusterBase.java index 7baf1a65..e12ff5a0 100644 --- a/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWithMiniClusterBase.java +++ b/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWithMiniClusterBase.java @@ -261,8 +261,10 @@ public void testUpdateSeen() throws Exception { assertThat(fileTypeRes.getStatusLine().getStatusCode(), is(200)); List fileTypeContent = IOUtils.readLines(fileTypeRes.getEntity().getContent()); long part_r_counts = fileTypeContent.stream().filter(s -> s.startsWith("PART_R")).count(); + long applog_counts = fileTypeContent.stream().filter(s -> s.startsWith("APP_LOG")).count(); assertThat(fileTypeContent.size(), is(greaterThan(0))); assertThat(part_r_counts, is(greaterThan(0L))); + assertThat(applog_counts, is(greaterThan(0L))); } protected void addFiles(int numOfFiles, long sleepBetweenMs) throws Exception { @@ -276,7 +278,7 @@ protected void addFiles(int numOfFiles, long sleepBetweenMs) throws Exception { dirPath = dirPath.suffix("/dir" + dirNumber3); fileSystem.mkdirs(dirPath); Path filePath = dirPath.suffix("/file" + i); - int fileType = RANDOM.nextInt(6); + int fileType = RANDOM.nextInt(7); switch (fileType) { case 0: filePath = filePath.suffix(".zip"); @@ -295,14 +297,14 @@ protected void addFiles(int numOfFiles, long sleepBetweenMs) throws Exception { break; case 5: filePath = dirPath.suffix("/part-r-" + i); + break; + case 6: + filePath = filePath.suffix("_45454"); default: break; } int fileSize = RANDOM.nextInt(4); switch (fileSize) { - case 0: - DFSTestUtil.writeFile(fileSystem, filePath, ""); - break; case 1: DFSTestUtil.writeFile(fileSystem, filePath, new String(TINY_FILE_BYTES)); break; @@ -312,6 +314,7 @@ protected void addFiles(int numOfFiles, long sleepBetweenMs) throws Exception { case 3: DFSTestUtil.writeFile(fileSystem, filePath, new String(MEDIUM_FILE_BYTES)); break; + case 0: default: DFSTestUtil.writeFile(fileSystem, filePath, ""); break; @@ -321,14 +324,13 @@ protected void addFiles(int numOfFiles, long sleepBetweenMs) throws Exception { } int user = RANDOM.nextInt(3); switch (user) { - case 0: - break; case 1: fileSystem.setOwner(filePath, USERS[0], USERS[0]); break; case 2: fileSystem.setOwner(filePath, USERS[1], USERS[1]); break; + case 0: default: break; }