Merge pull request #515 from Hurence/feature/LOGISLAND-483

Feature/logisland 483
Hurence · Nov 28, 2019 · a54d7e5 · a54d7e5
2 parents aae8286 + e6ba2ca
commit a54d7e5
Show file tree

Hide file tree

Showing 96 changed files with 4,011 additions and 5,283 deletions.
diff --git a/logisland-documentation/release.rst → RELEASING.rst b/logisland-documentation/release.rst → RELEASING.rst
diff --git a/...s/logisland-processor-common/src/main/java/com/hurence/logisland/processor/AddFields.java b/...s/logisland-processor-common/src/main/java/com/hurence/logisland/processor/AddFields.java
@@ -17,9 +17,7 @@
 
 import com.hurence.logisland.annotation.behavior.DynamicProperties;
 import com.hurence.logisland.annotation.behavior.DynamicProperty;
-import com.hurence.logisland.annotation.documentation.CapabilityDescription;
-import com.hurence.logisland.annotation.documentation.ExtraDetailFile;
-import com.hurence.logisland.annotation.documentation.Tags;
+import com.hurence.logisland.annotation.documentation.*;
 import com.hurence.logisland.component.AllowableValue;
 import com.hurence.logisland.component.InitializationException;
 import com.hurence.logisland.component.PropertyDescriptor;
@@ -34,6 +32,7 @@
 
 import java.util.*;
 
+@Category(ComponentCategory.PROCESSING)
 @Tags({"record", "fields", "Add"})
 @CapabilityDescription("Add one or more field to records")
 @ExtraDetailFile("./details/common-processors/AddFields-Detail.rst")
@@ -126,7 +125,7 @@ protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String
 
 
     @Override
-    public void init(ProcessContext context) throws InitializationException {
+    public void init(ProcessContext context)  throws InitializationException {
         super.init(context);
         initDynamicProperties(context);
         this.conflictPolicy = context.getPropertyValue(CONFLICT_RESOLUTION_POLICY).asString();

diff --git a/...logisland-processor-common/src/main/java/com/hurence/logisland/processor/ApplyRegexp.java b/...logisland-processor-common/src/main/java/com/hurence/logisland/processor/ApplyRegexp.java
@@ -16,27 +16,19 @@
 package com.hurence.logisland.processor;
 
 import com.hurence.logisland.annotation.behavior.DynamicProperty;
-import com.hurence.logisland.annotation.documentation.CapabilityDescription;
-import com.hurence.logisland.annotation.documentation.ExtraDetailFile;
-import com.hurence.logisland.annotation.documentation.SeeAlso;
-import com.hurence.logisland.annotation.documentation.Tags;
+import com.hurence.logisland.annotation.documentation.*;
 import com.hurence.logisland.component.AllowableValue;
 import com.hurence.logisland.component.PropertyDescriptor;
-import com.hurence.logisland.record.FieldDictionary;
-import com.hurence.logisland.record.FieldType;
 import com.hurence.logisland.record.Record;
-import com.hurence.logisland.record.StandardRecord;
-import com.hurence.logisland.util.time.DateUtil;
 import com.hurence.logisland.validator.StandardValidators;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.text.ParseException;
 import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-
+@Category(ComponentCategory.PROCESSING)
 @Tags({"parser", "regex", "log", "record"})
 @CapabilityDescription("This processor is used to create a new set of fields from one field (using regexp).")
 @SeeAlso(value = {ApplyRegexp.class}, classNames = {"com.hurence.logisland.processor.RegexpProcessor"})

diff --git a/...logisland-processor-common/src/main/java/com/hurence/logisland/processor/DebugStream.java b/...logisland-processor-common/src/main/java/com/hurence/logisland/processor/DebugStream.java
@@ -15,9 +15,7 @@
  */
 package com.hurence.logisland.processor;
 
-import com.hurence.logisland.annotation.documentation.CapabilityDescription;
-import com.hurence.logisland.annotation.documentation.ExtraDetailFile;
-import com.hurence.logisland.annotation.documentation.Tags;
+import com.hurence.logisland.annotation.documentation.*;
 import com.hurence.logisland.component.AllowableValue;
 import com.hurence.logisland.component.InitializationException;
 import com.hurence.logisland.component.PropertyDescriptor;
@@ -36,6 +34,7 @@
 import java.util.Collections;
 import java.util.List;
 
+@Category(ComponentCategory.UTILS)
 @Tags({"record", "debug"})
 @CapabilityDescription("This is a processor that logs incoming records")
 @ExtraDetailFile("./details/common-processors/DebugStream-Detail.rst")
@@ -69,7 +68,7 @@ public final List<PropertyDescriptor> getSupportedPropertyDescriptors() {
 
 
     @Override
-    public void init(ProcessContext context) throws InitializationException {
+    public void init(ProcessContext context)  throws InitializationException {
         super.init(context);
         if (memBean == null) {
             memBean = ManagementFactory.getMemoryMXBean();

diff --git a/...land-processor-common/src/main/java/com/hurence/logisland/processor/EvaluateJsonPath.java b/...land-processor-common/src/main/java/com/hurence/logisland/processor/EvaluateJsonPath.java
@@ -16,9 +16,7 @@
 package com.hurence.logisland.processor;
 
 import com.hurence.logisland.annotation.behavior.DynamicProperty;
-import com.hurence.logisland.annotation.documentation.CapabilityDescription;
-import com.hurence.logisland.annotation.documentation.ExtraDetailFile;
-import com.hurence.logisland.annotation.documentation.Tags;
+import com.hurence.logisland.annotation.documentation.*;
 import com.hurence.logisland.component.PropertyDescriptor;
 import com.hurence.logisland.record.FieldDictionary;
 import com.hurence.logisland.record.FieldType;
@@ -38,7 +36,7 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicReference;
 
-
+@Category(ComponentCategory.PARSING)
 @Tags({"JSON", "evaluate", "JsonPath"})
 @CapabilityDescription("Evaluates one or more JsonPath expressions against the content of a FlowFile. "
         + "The results of those expressions are assigned to Records Fields "

diff --git a/...sland-processor-common/src/main/java/com/hurence/logisland/processor/ExpandMapFields.java b/...sland-processor-common/src/main/java/com/hurence/logisland/processor/ExpandMapFields.java
@@ -15,9 +15,7 @@
  */
 package com.hurence.logisland.processor;
 
-import com.hurence.logisland.annotation.documentation.CapabilityDescription;
-import com.hurence.logisland.annotation.documentation.ExtraDetailFile;
-import com.hurence.logisland.annotation.documentation.Tags;
+import com.hurence.logisland.annotation.documentation.*;
 import com.hurence.logisland.component.AllowableValue;
 import com.hurence.logisland.component.PropertyDescriptor;
 import com.hurence.logisland.record.Field;
@@ -29,6 +27,7 @@
 
 import java.util.*;
 
+@Category(ComponentCategory.PROCESSING)
 @Tags({"record", "fields", "Expand", "Map"})
 @CapabilityDescription("Expands the content of a MAP field to the root.")
 @ExtraDetailFile("./details/common-processors/ExpandMapFields-Detail.rst")

diff --git a/...ors/logisland-processor-common/src/main/java/com/hurence/logisland/processor/FlatMap.java b/...ors/logisland-processor-common/src/main/java/com/hurence/logisland/processor/FlatMap.java
@@ -15,16 +15,15 @@
  */
 package com.hurence.logisland.processor;
 
-import com.hurence.logisland.annotation.documentation.CapabilityDescription;
-import com.hurence.logisland.annotation.documentation.ExtraDetailFile;
-import com.hurence.logisland.annotation.documentation.Tags;
+import com.hurence.logisland.annotation.documentation.*;
 import com.hurence.logisland.component.PropertyDescriptor;
 import com.hurence.logisland.record.*;
 import com.hurence.logisland.validator.StandardValidators;
 
 import java.util.*;
 import java.util.stream.Collectors;
 
+@Category(ComponentCategory.PROCESSING)
 @Tags({"record", "fields", "flatmap", "flatten"})
 @CapabilityDescription("Converts each field records into a single flatten record...")
 @ExtraDetailFile("./details/common-processors/FlatMap-Detail.rst")

diff --git a/...-processor-common/src/main/java/com/hurence/logisland/processor/GenerateRandomRecord.java b/...-processor-common/src/main/java/com/hurence/logisland/processor/GenerateRandomRecord.java
@@ -15,9 +15,7 @@
  */
 package com.hurence.logisland.processor;
 
-import com.hurence.logisland.annotation.documentation.CapabilityDescription;
-import com.hurence.logisland.annotation.documentation.ExtraDetailFile;
-import com.hurence.logisland.annotation.documentation.Tags;
+import com.hurence.logisland.annotation.documentation.*;
 import com.hurence.logisland.component.PropertyDescriptor;
 import com.hurence.logisland.record.Record;
 import com.hurence.logisland.util.avro.eventgenerator.DataGenerator;
@@ -31,6 +29,7 @@
 import java.util.Collections;
 import java.util.List;
 
+@Category(ComponentCategory.PROCESSING)
 @Tags({"record", "avro", "generator"})
 @CapabilityDescription("This is a processor that make random records given an Avro schema")
 @ExtraDetailFile("./details/common-processors/GenerateRandomRecord-Detail.rst")

diff --git a/...rs/logisland-processor-common/src/main/java/com/hurence/logisland/processor/ModifyId.java b/...rs/logisland-processor-common/src/main/java/com/hurence/logisland/processor/ModifyId.java
@@ -16,9 +16,7 @@
 package com.hurence.logisland.processor;
 
 import com.google.common.collect.Lists;
-import com.hurence.logisland.annotation.documentation.CapabilityDescription;
-import com.hurence.logisland.annotation.documentation.ExtraDetailFile;
-import com.hurence.logisland.annotation.documentation.Tags;
+import com.hurence.logisland.annotation.documentation.*;
 import com.hurence.logisland.component.AllowableValue;
 import com.hurence.logisland.component.InitializationException;
 import com.hurence.logisland.component.PropertyDescriptor;
@@ -36,6 +34,7 @@
 import java.security.NoSuchAlgorithmException;
 import java.util.*;
 
+@Category(ComponentCategory.PROCESSING)
 @Tags({"record", "id", "idempotent", "generate", "modify"})
 @CapabilityDescription("modify id of records or generate it following defined rules")
 @ExtraDetailFile("./details/common-processors/ModifyId-Detail.rst")
@@ -161,7 +160,7 @@ protected Collection<ValidationResult> customValidate(ValidationContext context)
     private IdBuilder idBuilder = null;
 
     @Override
-    public void init(ProcessContext context) throws InitializationException {
+    public void init(ProcessContext context)   throws InitializationException {
         super.init(context);
         if (context.getPropertyValue(STRATEGY).isSet()) {
             if (context.getPropertyValue(STRATEGY).getRawValue().equals(RANDOM_UUID_STRATEGY.getValue())) {

diff --git a/...sland-processor-common/src/main/java/com/hurence/logisland/processor/NormalizeFields.java b/...sland-processor-common/src/main/java/com/hurence/logisland/processor/NormalizeFields.java
@@ -16,19 +16,16 @@
 package com.hurence.logisland.processor;
 
 import com.hurence.logisland.annotation.behavior.DynamicProperty;
-import com.hurence.logisland.annotation.documentation.CapabilityDescription;
-import com.hurence.logisland.annotation.documentation.ExtraDetailFile;
-import com.hurence.logisland.annotation.documentation.Tags;
+import com.hurence.logisland.annotation.documentation.*;
 import com.hurence.logisland.component.AllowableValue;
 import com.hurence.logisland.component.PropertyDescriptor;
 import com.hurence.logisland.record.Field;
 import com.hurence.logisland.record.Record;
 import com.hurence.logisland.validator.StandardValidators;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.util.*;
 
+@Category(ComponentCategory.PROCESSING)
 @Tags({"record", "fields", "normalizer"})
 @CapabilityDescription("Changes the name of a field according to a provided name mapping...")
 @DynamicProperty(name = "alternative mapping",

diff --git a/...sland-processor-common/src/main/java/com/hurence/logisland/processor/ParseProperties.java b/...sland-processor-common/src/main/java/com/hurence/logisland/processor/ParseProperties.java
@@ -17,9 +17,7 @@
 
 import com.google.common.collect.Lists;
 import com.hurence.logisland.annotation.behavior.DynamicProperty;
-import com.hurence.logisland.annotation.documentation.CapabilityDescription;
-import com.hurence.logisland.annotation.documentation.ExtraDetailFile;
-import com.hurence.logisland.annotation.documentation.Tags;
+import com.hurence.logisland.annotation.documentation.*;
 import com.hurence.logisland.component.PropertyDescriptor;
 import com.hurence.logisland.record.Field;
 import com.hurence.logisland.record.FieldType;
@@ -33,6 +31,7 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+@Category(ComponentCategory.PARSING)
 @Tags({"record", "properties", "parser"})
 @CapabilityDescription("Parse a field made of key=value fields separated by spaces\n" +
         "a string like \"a=1 b=2 c=3\" will add a,b & c fields, respectively with values 1,2 & 3 to the current Record")

diff --git a/...processor-common/src/main/java/com/hurence/logisland/processor/SelectDistinctRecords.java b/...processor-common/src/main/java/com/hurence/logisland/processor/SelectDistinctRecords.java
@@ -16,9 +16,7 @@
 package com.hurence.logisland.processor;
 
 import com.google.common.collect.Lists;
-import com.hurence.logisland.annotation.documentation.CapabilityDescription;
-import com.hurence.logisland.annotation.documentation.ExtraDetailFile;
-import com.hurence.logisland.annotation.documentation.Tags;
+import com.hurence.logisland.annotation.documentation.*;
 import com.hurence.logisland.component.PropertyDescriptor;
 import com.hurence.logisland.record.FieldDictionary;
 import com.hurence.logisland.record.Record;
@@ -35,6 +33,7 @@
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
+@Category(ComponentCategory.PROCESSING)
 @Tags({"record", "fields", "remove", "delete"})
 @CapabilityDescription("Keep only distinct records based on a given field")
 @ExtraDetailFile("./details/common-processors/SelectDistinctRecords-Detail.rst")

diff --git a/.../logisland-processor-common/src/main/java/com/hurence/logisland/processor/SplitField.java b/.../logisland-processor-common/src/main/java/com/hurence/logisland/processor/SplitField.java
@@ -16,10 +16,7 @@
 package com.hurence.logisland.processor;
 
 import com.hurence.logisland.annotation.behavior.DynamicProperty;
-import com.hurence.logisland.annotation.documentation.CapabilityDescription;
-import com.hurence.logisland.annotation.documentation.ExtraDetailFile;
-import com.hurence.logisland.annotation.documentation.SeeAlso;
-import com.hurence.logisland.annotation.documentation.Tags;
+import com.hurence.logisland.annotation.documentation.*;
 import com.hurence.logisland.component.AllowableValue;
 import com.hurence.logisland.component.InitializationException;
 import com.hurence.logisland.component.PropertyDescriptor;
@@ -33,6 +30,7 @@
 import java.util.regex.Pattern;
 
 
+@Category(ComponentCategory.PARSING)
 @Tags({"parser", "split", "log", "record"})
 @CapabilityDescription("This processor is used to create a new set of fields from one field (using split).")
 @SeeAlso(value = {SplitField.class}, classNames = {"com.hurence.logisland.processor.SplitField"})
@@ -112,7 +110,7 @@ protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String
     }
 
     @Override
-    public void init(final ProcessContext context) throws InitializationException {
+    public void init(final ProcessContext context) throws InitializationException{
         super.init(context);
         this.fieldsNameMapping = getFieldsNameMapping(context);
         this.nbSplitLimit = context.getPropertyValue(NB_SPLIT_LIMIT).asInteger();