Add StreamCategoryFilter and stream_category to StreamDTO

graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackend.java

@@ -59,6 +59,7 @@
 import org.graylog2.indexer.ranges.IndexRange;
 import org.graylog2.plugin.Message;
 import org.graylog2.plugin.indexer.searches.timeranges.TimeRange;
+import org.graylog2.streams.StreamService;
 import org.joda.time.DateTimeZone;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -76,7 +77,6 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-import java.util.stream.Collectors;
 
 public class ElasticsearchBackend implements QueryBackend<ESGeneratedQueryContext> {
     private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchBackend.class);
@@ -88,6 +88,7 @@ public class ElasticsearchBackend implements QueryBackend<ESGeneratedQueryContex
     private final UsedSearchFiltersToQueryStringsMapper usedSearchFiltersToQueryStringsMapper;
     private final boolean allowLeadingWildcard;
     private final StatsCollector<QueryExecutionStats> executionStatsCollector;
+    private final StreamService streamService;
 
     @Inject
     public ElasticsearchBackend(Map<String, Provider<ESSearchTypeHandler<? extends SearchType>>> elasticsearchSearchTypeHandlers,
@@ -96,6 +97,7 @@ public ElasticsearchBackend(Map<String, Provider<ESSearchTypeHandler<? extends S
                                 ESGeneratedQueryContext.Factory queryContextFactory,
                                 UsedSearchFiltersToQueryStringsMapper usedSearchFiltersToQueryStringsMapper,
                                 StatsCollector<QueryExecutionStats> executionStatsCollector,
+                                StreamService streamService,
                                 @Named("allow_leading_wildcard_searches") boolean allowLeadingWildcard) {
         this.elasticsearchSearchTypeHandlers = elasticsearchSearchTypeHandlers;
         this.client = client;
@@ -104,6 +106,7 @@ public ElasticsearchBackend(Map<String, Provider<ESSearchTypeHandler<? extends S
         this.queryContextFactory = queryContextFactory;
         this.usedSearchFiltersToQueryStringsMapper = usedSearchFiltersToQueryStringsMapper;
         this.executionStatsCollector = executionStatsCollector;
+        this.streamService = streamService;
         this.allowLeadingWildcard = allowLeadingWildcard;
     }
 
@@ -160,7 +163,7 @@ public ESGeneratedQueryContext generate(Query query, Set<SearchError> validation
                     final SearchSourceBuilder searchTypeSourceBuilder = queryContext.searchSourceBuilder(searchType);
 
                     final Set<String> effectiveStreamIds = query.effectiveStreams(searchType);
-
+                    effectiveStreamIds.addAll(streamService.mapCategoriesToIds(query.usedStreamCategories()));
                     final BoolQueryBuilder searchTypeOverrides = QueryBuilders.boolQuery()
                             .must(searchTypeSourceBuilder.query())
                             .must(
@@ -239,7 +242,9 @@ public QueryResult doRun(SearchJob job, Query query, ESGeneratedQueryContext que
         LOG.debug("Running query {} for job {}", query.id(), job.getId());
         final HashMap<String, SearchType.Result> resultsMap = Maps.newHashMap();
 
-        final Set<String> affectedIndices = indexLookup.indexNamesForStreamsInTimeRange(query.usedStreamIds(), query.timerange());
+        final Set<String> usedStreams = query.usedStreamIds();
+        usedStreams.addAll(streamService.mapCategoriesToIds(query.usedStreamCategories()));
+        final Set<String> affectedIndices = indexLookup.indexNamesForStreamsInTimeRange(usedStreams, query.timerange());
 
         final Map<String, SearchSourceBuilder> searchTypeQueries = queryContext.searchTypeQueries();
         final List<String> searchTypeIds = new ArrayList<>(searchTypeQueries.keySet());

graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/OpenSearchBackend.java

@@ -61,6 +61,7 @@
 import org.graylog2.plugin.Message;
 import org.graylog2.plugin.indexer.searches.timeranges.TimeRange;
 import org.graylog2.plugin.streams.Stream;
+import org.graylog2.streams.StreamService;
 import org.joda.time.DateTimeZone;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -77,7 +78,6 @@
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
 public class OpenSearchBackend implements QueryBackend<OSGeneratedQueryContext> {
     private static final Logger LOG = LoggerFactory.getLogger(OpenSearchBackend.class);
@@ -89,6 +89,7 @@ public class OpenSearchBackend implements QueryBackend<OSGeneratedQueryContext>
     private final UsedSearchFiltersToQueryStringsMapper usedSearchFiltersToQueryStringsMapper;
     private final boolean allowLeadingWildcard;
     private final StatsCollector<QueryExecutionStats> executionStatsCollector;
+    private final StreamService streamService;
 
     @Inject
     public OpenSearchBackend(Map<String, Provider<OSSearchTypeHandler<? extends SearchType>>> elasticsearchSearchTypeHandlers,
@@ -97,6 +98,7 @@ public OpenSearchBackend(Map<String, Provider<OSSearchTypeHandler<? extends Sear
                              OSGeneratedQueryContext.Factory queryContextFactory,
                              UsedSearchFiltersToQueryStringsMapper usedSearchFiltersToQueryStringsMapper,
                              StatsCollector<QueryExecutionStats> executionStatsCollector,
+                             StreamService streamService,
                              @Named("allow_leading_wildcard_searches") boolean allowLeadingWildcard) {
         this.openSearchSearchTypeHandlers = elasticsearchSearchTypeHandlers;
         this.client = client;
@@ -105,6 +107,7 @@ public OpenSearchBackend(Map<String, Provider<OSSearchTypeHandler<? extends Sear
         this.queryContextFactory = queryContextFactory;
         this.usedSearchFiltersToQueryStringsMapper = usedSearchFiltersToQueryStringsMapper;
         this.executionStatsCollector = executionStatsCollector;
+        this.streamService = streamService;
         this.allowLeadingWildcard = allowLeadingWildcard;
     }
 
@@ -159,7 +162,7 @@ public OSGeneratedQueryContext generate(Query query, Set<SearchError> validation
                     final SearchSourceBuilder searchTypeSourceBuilder = queryContext.searchSourceBuilder(searchType);
 
                     final Set<String> effectiveStreamIds = query.effectiveStreams(searchType);
-
+                    effectiveStreamIds.addAll(streamService.mapCategoriesToIds(query.usedStreamCategories()));
                     final BoolQueryBuilder searchTypeOverrides = QueryBuilders.boolQuery()
                             .must(searchTypeSourceBuilder.query())
                             .must(
@@ -242,7 +245,9 @@ public QueryResult doRun(SearchJob job, Query query, OSGeneratedQueryContext que
         LOG.debug("Running query {} for job {}", query.id(), job.getId());
         final HashMap<String, SearchType.Result> resultsMap = Maps.newHashMap();
 
-        final Set<String> affectedIndices = indexLookup.indexNamesForStreamsInTimeRange(query.usedStreamIds(), query.timerange());
+        final Set<String> usedStreams = query.usedStreamIds();
+        usedStreams.addAll(streamService.mapCategoriesToIds(query.usedStreamCategories()));
+        final Set<String> affectedIndices = indexLookup.indexNamesForStreamsInTimeRange(usedStreams, query.timerange());
 
         final Map<String, SearchSourceBuilder> searchTypeQueries = queryContext.searchTypeQueries();
         final List<String> searchTypeIds = new ArrayList<>(searchTypeQueries.keySet());

graylog2-server/src/main/java/org/graylog/plugins/views/ViewsBindings.java

@@ -56,6 +56,7 @@
 import org.graylog.plugins.views.search.filter.AndFilter;
 import org.graylog.plugins.views.search.filter.OrFilter;
 import org.graylog.plugins.views.search.filter.QueryStringFilter;
+import org.graylog.plugins.views.search.filter.StreamCategoryFilter;
 import org.graylog.plugins.views.search.filter.StreamFilter;
 import org.graylog.plugins.views.search.querystrings.LastUsedQueryStringsService;
 import org.graylog.plugins.views.search.querystrings.MongoLastUsedQueryStringsService;
@@ -177,6 +178,7 @@ protected void configure() {
         registerJacksonSubtype(AndFilter.class);
         registerJacksonSubtype(OrFilter.class);
         registerJacksonSubtype(StreamFilter.class);
+        registerJacksonSubtype(StreamCategoryFilter.class);
         registerJacksonSubtype(QueryStringFilter.class);
 
         // query backends for jackson

graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java

@@ -32,6 +32,7 @@
 import org.graylog.plugins.views.search.engine.BackendQuery;
 import org.graylog.plugins.views.search.engine.EmptyTimeRange;
 import org.graylog.plugins.views.search.filter.AndFilter;
+import org.graylog.plugins.views.search.filter.StreamCategoryFilter;
 import org.graylog.plugins.views.search.filter.StreamFilter;
 import org.graylog.plugins.views.search.rest.ExecutionState;
 import org.graylog.plugins.views.search.rest.ExecutionStateGlobalOverride;
@@ -210,6 +211,20 @@ public Set<String> usedStreamIds() {
                 .orElse(Collections.emptySet());
     }
 
+    @SuppressWarnings("UnstableApiUsage")
+    public Set<String> usedStreamCategories() {
+        return Optional.ofNullable(filter())
+                .map(optFilter -> {
+                    final Traverser<Filter> filterTraverser = Traverser.forTree(filter -> firstNonNull(filter.filters(), Collections.emptySet()));
+                    return StreamSupport.stream(filterTraverser.breadthFirst(optFilter).spliterator(), false)
+                            .filter(filter -> filter instanceof StreamCategoryFilter)
+                            .map(streamFilter -> ((StreamCategoryFilter) streamFilter).category())
+                            .filter(Objects::nonNull)
+                            .collect(toSet());
+                })
+                .orElse(Collections.emptySet());
+    }
+
     public Set<String> streamIdsForPermissionsCheck() {
         final Set<String> searchTypeStreamIds = searchTypes().stream()
                 .map(SearchType::streams)
@@ -219,7 +234,7 @@ public Set<String> streamIdsForPermissionsCheck() {
     }
 
     public boolean hasStreams() {
-        return !usedStreamIds().isEmpty();
+        return !(usedStreamIds().isEmpty() && usedStreamCategories().isEmpty());
     }
 
     public boolean hasReferencedStreamFilters() {

graylog2-server/src/main/java/org/graylog/plugins/views/search/Search.java

@@ -168,12 +168,24 @@ public Set<String> usedStreamIds() {
         return Sets.union(queryStreamIds, searchTypeStreamIds);
     }
 
+    public Set<String> usedStreamCategories() {
+        return queries().stream()
+                .map(Query::usedStreamCategories)
+                .reduce(Collections.emptySet(), Sets::union);
+    }
+
     public Set<String> streamIdsForPermissionsCheck() {
         return queries().stream()
                 .map(Query::streamIdsForPermissionsCheck)
                 .reduce(Collections.emptySet(), Sets::union);
     }
 
+    public Set<String> streamCategoriesForPermissionsCheck() {
+        return queries().stream()
+                .map(Query::usedStreamCategories)
+                .reduce(Collections.emptySet(), Sets::union);
+    }
+
     public Query queryForSearchType(String searchTypeId) {
         return queries().stream()
                 .filter(q -> q.hasSearchType(searchTypeId))

graylog2-server/src/main/java/org/graylog/plugins/views/search/export/CommandFactory.java

@@ -27,17 +27,22 @@
 import org.graylog2.decorators.Decorator;
 import org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange;
 import org.graylog2.plugin.indexer.searches.timeranges.TimeRange;
+import org.graylog2.streams.StreamService;
 
 import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 public class CommandFactory {
     private final QueryStringDecorators queryStringDecorator;
+    private final StreamService streamService;
 
     @Inject
-    public CommandFactory(QueryStringDecorators queryStringDecorator) {
+    public CommandFactory(QueryStringDecorators queryStringDecorator,
+                          StreamService streamService) {
         this.queryStringDecorator = queryStringDecorator;
+        this.streamService = streamService;
     }
 
     public ExportMessagesCommand buildFromRequest(MessagesRequest request) {
@@ -56,6 +61,8 @@ public ExportMessagesCommand buildFromRequest(MessagesRequest request) {
 
     public ExportMessagesCommand buildWithSearchOnly(Search search, ResultFormat resultFormat) {
         Query query = queryFrom(search);
+        final Set<String> queryStreamIds = query.usedStreamIds();
+        queryStreamIds.addAll(streamService.mapCategoriesToIds(query.usedStreamCategories()));
 
         return builderFrom(resultFormat)
                 .timeRange(resultFormat.timerange().orElse(toAbsolute(query.timerange())))
@@ -65,7 +72,7 @@ public ExportMessagesCommand buildWithSearchOnly(Search search, ResultFormat res
                         .flatMap(List::stream)
                         .collect(Collectors.toList())
                 )
-                .streams(query.usedStreamIds())
+                .streams(queryStreamIds)
                 .build();
     }
 

graylog2-server/src/main/java/org/graylog/plugins/views/search/filter/StreamCategoryFilter.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.views.search.filter;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.google.auto.value.AutoValue;
+import org.graylog.plugins.views.search.Filter;
+
+import javax.annotation.Nullable;
+import java.util.Set;
+
+@AutoValue
+@JsonTypeName(StreamCategoryFilter.NAME)
+@JsonDeserialize(builder = StreamCategoryFilter.Builder.class)
+public abstract class StreamCategoryFilter implements Filter {
+    public static final String NAME = "stream_category";
+
+    @Override
+    @JsonProperty
+    public abstract String type();
+
+    @Override
+    @Nullable
+    @JsonProperty
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    public abstract Set<Filter> filters();
+
+    @JsonProperty("category")
+    public abstract String category();
+
+    public static Builder builder() {
+        return Builder.create();
+    }
+
+    public abstract Builder toBuilder();
+
+    @Override
+    public Filter.Builder toGenericBuilder() {
+        return toBuilder();
+    }
+
+    @AutoValue.Builder
+    public abstract static class Builder implements Filter.Builder {
+        @JsonProperty
+        public abstract Builder type(String type);
+
+        @JsonProperty
+        public abstract Builder filters(@Nullable Set<Filter> filters);
+
+        @JsonProperty("category")
+        public abstract Builder category(String category);
+
+        public abstract StreamCategoryFilter build();
+
+        @JsonCreator
+        public static Builder create() {
+            return new AutoValue_StreamCategoryFilter.Builder().type(NAME);
+        }
+    }
+}

graylog2-server/src/main/java/org/graylog2/plugin/streams/Stream.java

@@ -89,6 +89,8 @@ public static MatchingType valueOfOrDefault(String name) {
 
     String getContentPack();
 
+    List<String> getCategories();
+
     void setTitle(String title);
 
     void setDescription(String description);
@@ -99,6 +101,8 @@ public static MatchingType valueOfOrDefault(String name) {
 
     void setMatchingType(MatchingType matchingType);
 
+    void setCategories(List<String> categories);
+
     Boolean isPaused();
 
     Map<String, Object> asMap(List<StreamRule> streamRules);

graylog2-server/src/main/java/org/graylog2/streams/StreamDTO.java

@@ -32,6 +32,7 @@
 import javax.annotation.Nullable;
 import java.util.Collection;
 import java.util.Date;
+import java.util.List;
 
 @AutoValue
 @WithBeanGetter
@@ -54,6 +55,7 @@ public abstract class StreamDTO {
     public static final String FIELD_INDEX_SET_ID = "index_set_id";
     public static final String EMBEDDED_ALERT_CONDITIONS = "alert_conditions";
     public static final String FIELD_IS_EDITABLE = "is_editable";
+    public static final String FIELD_CATEGORIES = "categories";
     public static final Stream.MatchingType DEFAULT_MATCHING_TYPE = Stream.MatchingType.AND;
 
     @JsonProperty("id")
@@ -114,6 +116,10 @@ public abstract class StreamDTO {
     @JsonProperty(FIELD_IS_EDITABLE)
     public abstract boolean isEditable();
 
+    @JsonProperty(FIELD_CATEGORIES)
+    @Nullable
+    public abstract List<String> categories();
+
     public abstract Builder toBuilder();
 
     static Builder builder() {
@@ -128,7 +134,8 @@ public static Builder create() {
                     .matchingType(DEFAULT_MATCHING_TYPE.toString())
                     .isDefault(false)
                     .isEditable(false)
-                    .removeMatchesFromDefaultStream(false);
+                    .removeMatchesFromDefaultStream(false)
+                    .categories(List.of());
         }
 
         @JsonProperty(FIELD_ID)
@@ -181,6 +188,9 @@ public static Builder create() {
         @JsonProperty(FIELD_IS_EDITABLE)
         public abstract Builder isEditable(boolean isEditable);
 
+        @JsonProperty(FIELD_CATEGORIES)
+        public abstract Builder categories(List<String> categories);
+
         public abstract String id();
 
         public abstract StreamDTO autoBuild();
@@ -206,6 +216,7 @@ public static StreamDTO fromDocument(Document document) {
                 .creatorUserId(document.getString(FIELD_CREATOR_USER_ID))
                 .indexSetId(document.getString(FIELD_INDEX_SET_ID))
                 .outputs(document.getList(FIELD_OUTPUTS, ObjectId.class))
+                .categories(document.getList(FIELD_CATEGORIES, String.class))
                 .build();
     }
 }