From 0ec884ca001cf170571d560f915d996e89737cea Mon Sep 17 00:00:00 2001 From: Zack King Date: Wed, 7 Aug 2024 13:44:22 -0500 Subject: [PATCH 01/13] Add StreamCategoryFilter and stream_category to StreamDTO --- .../views/ElasticsearchBackend.java | 11 ++- .../views/searchtypes/ESEventList.java | 9 ++- .../views/searchtypes/ESMessageList.java | 4 + .../opensearch2/views/OpenSearchBackend.java | 11 ++- .../views/searchtypes/OSEventList.java | 9 ++- .../views/searchtypes/OSMessageList.java | 4 + .../graylog/plugins/views/ViewsBindings.java | 2 + .../graylog/plugins/views/search/Query.java | 17 +++- .../graylog/plugins/views/search/Search.java | 12 +++ .../views/search/export/CommandFactory.java | 11 ++- .../search/filter/StreamCategoryFilter.java | 78 +++++++++++++++++++ .../org/graylog2/plugin/streams/Stream.java | 4 + .../java/org/graylog2/streams/StreamDTO.java | 13 +++- .../java/org/graylog2/streams/StreamImpl.java | 13 ++++ .../org/graylog2/streams/StreamService.java | 2 + .../graylog2/streams/StreamServiceImpl.java | 11 +++ 16 files changed, 197 insertions(+), 14 deletions(-) create mode 100644 graylog2-server/src/main/java/org/graylog/plugins/views/search/filter/StreamCategoryFilter.java diff --git a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackend.java b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackend.java index 9f549d13a74f..3d3a477ee102 100644 --- a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackend.java +++ b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackend.java @@ -59,6 +59,7 @@ import org.graylog2.indexer.ranges.IndexRange; import org.graylog2.plugin.Message; import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; +import org.graylog2.streams.StreamService; import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,7 +77,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; public class ElasticsearchBackend implements QueryBackend { private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchBackend.class); @@ -88,6 +88,7 @@ public class ElasticsearchBackend implements QueryBackend executionStatsCollector; + private final StreamService streamService; @Inject public ElasticsearchBackend(Map>> elasticsearchSearchTypeHandlers, @@ -96,6 +97,7 @@ public ElasticsearchBackend(Map executionStatsCollector, + StreamService streamService, @Named("allow_leading_wildcard_searches") boolean allowLeadingWildcard) { this.elasticsearchSearchTypeHandlers = elasticsearchSearchTypeHandlers; this.client = client; @@ -104,6 +106,7 @@ public ElasticsearchBackend(Map validation final SearchSourceBuilder searchTypeSourceBuilder = queryContext.searchSourceBuilder(searchType); final Set effectiveStreamIds = query.effectiveStreams(searchType); - + effectiveStreamIds.addAll(streamService.mapCategoriesToIds(query.usedStreamCategories())); final BoolQueryBuilder searchTypeOverrides = QueryBuilders.boolQuery() .must(searchTypeSourceBuilder.query()) .must( @@ -239,7 +242,9 @@ public QueryResult doRun(SearchJob job, Query query, ESGeneratedQueryContext que LOG.debug("Running query {} for job {}", query.id(), job.getId()); final HashMap resultsMap = Maps.newHashMap(); - final Set affectedIndices = indexLookup.indexNamesForStreamsInTimeRange(query.usedStreamIds(), query.timerange()); + final Set usedStreams = query.usedStreamIds(); + usedStreams.addAll(streamService.mapCategoriesToIds(query.usedStreamCategories())); + final Set affectedIndices = indexLookup.indexNamesForStreamsInTimeRange(usedStreams, query.timerange()); final Map searchTypeQueries = queryContext.searchTypeQueries(); final List searchTypeIds = new ArrayList<>(searchTypeQueries.keySet()); diff --git a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/searchtypes/ESEventList.java b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/searchtypes/ESEventList.java index 6daf563c5a65..8baf10c604d2 100644 --- a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/searchtypes/ESEventList.java +++ b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/searchtypes/ESEventList.java @@ -34,6 +34,7 @@ import org.graylog.shaded.elasticsearch7.org.elasticsearch.search.sort.FieldSortBuilder; import org.graylog.shaded.elasticsearch7.org.elasticsearch.search.sort.SortOrder; import org.graylog.storage.elasticsearch7.views.ESGeneratedQueryContext; +import org.graylog2.streams.StreamService; import java.util.List; import java.util.Map; @@ -43,17 +44,21 @@ public class ESEventList implements ESSearchTypeHandler { private final ObjectMapper objectMapper; + private final StreamService streamService; @Inject - public ESEventList(ObjectMapper objectMapper) { + public ESEventList(ObjectMapper objectMapper, StreamService streamService) { this.objectMapper = objectMapper; + this.streamService = streamService; } @Override public void doGenerateQueryPart(Query query, EventList eventList, ESGeneratedQueryContext queryContext) { + final Set queryStreamIds = query.usedStreamIds(); + queryStreamIds.addAll(streamService.mapCategoriesToIds(query.usedStreamCategories())); final Set effectiveStreams = eventList.streams().isEmpty() - ? query.usedStreamIds() + ? queryStreamIds : eventList.streams(); final var searchSourceBuilder = queryContext.searchSourceBuilder(eventList); diff --git a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/searchtypes/ESMessageList.java b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/searchtypes/ESMessageList.java index c12de5dc6781..0ff0aa9eec65 100644 --- a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/searchtypes/ESMessageList.java +++ b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/searchtypes/ESMessageList.java @@ -43,6 +43,7 @@ import org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange; import org.graylog2.rest.models.messages.responses.ResultMessageSummary; import org.graylog2.rest.resources.search.responses.SearchResponse; +import org.graylog2.streams.StreamService; import org.joda.time.DateTime; import java.util.ArrayList; @@ -60,14 +61,17 @@ public class ESMessageList implements ESSearchTypeHandler { private final LegacyDecoratorProcessor decoratorProcessor; private final ResultMessageFactory resultMessageFactory; + private final StreamService streamService; private final boolean allowHighlighting; @Inject public ESMessageList(LegacyDecoratorProcessor decoratorProcessor, ResultMessageFactory resultMessageFactory, + StreamService streamService, @Named("allow_highlighting") boolean allowHighlighting) { this.decoratorProcessor = decoratorProcessor; this.resultMessageFactory = resultMessageFactory; + this.streamService = streamService; this.allowHighlighting = allowHighlighting; } diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/OpenSearchBackend.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/OpenSearchBackend.java index 78fe70799a93..8d6d553da127 100644 --- a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/OpenSearchBackend.java +++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/OpenSearchBackend.java @@ -61,6 +61,7 @@ import org.graylog2.plugin.Message; import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; import org.graylog2.plugin.streams.Stream; +import org.graylog2.streams.StreamService; import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,7 +78,6 @@ import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; public class OpenSearchBackend implements QueryBackend { private static final Logger LOG = LoggerFactory.getLogger(OpenSearchBackend.class); @@ -89,6 +89,7 @@ public class OpenSearchBackend implements QueryBackend private final UsedSearchFiltersToQueryStringsMapper usedSearchFiltersToQueryStringsMapper; private final boolean allowLeadingWildcard; private final StatsCollector executionStatsCollector; + private final StreamService streamService; @Inject public OpenSearchBackend(Map>> elasticsearchSearchTypeHandlers, @@ -97,6 +98,7 @@ public OpenSearchBackend(Map executionStatsCollector, + StreamService streamService, @Named("allow_leading_wildcard_searches") boolean allowLeadingWildcard) { this.openSearchSearchTypeHandlers = elasticsearchSearchTypeHandlers; this.client = client; @@ -105,6 +107,7 @@ public OpenSearchBackend(Map validation final SearchSourceBuilder searchTypeSourceBuilder = queryContext.searchSourceBuilder(searchType); final Set effectiveStreamIds = query.effectiveStreams(searchType); - + effectiveStreamIds.addAll(streamService.mapCategoriesToIds(query.usedStreamCategories())); final BoolQueryBuilder searchTypeOverrides = QueryBuilders.boolQuery() .must(searchTypeSourceBuilder.query()) .must( @@ -242,7 +245,9 @@ public QueryResult doRun(SearchJob job, Query query, OSGeneratedQueryContext que LOG.debug("Running query {} for job {}", query.id(), job.getId()); final HashMap resultsMap = Maps.newHashMap(); - final Set affectedIndices = indexLookup.indexNamesForStreamsInTimeRange(query.usedStreamIds(), query.timerange()); + final Set usedStreams = query.usedStreamIds(); + usedStreams.addAll(streamService.mapCategoriesToIds(query.usedStreamCategories())); + final Set affectedIndices = indexLookup.indexNamesForStreamsInTimeRange(usedStreams, query.timerange()); final Map searchTypeQueries = queryContext.searchTypeQueries(); final List searchTypeIds = new ArrayList<>(searchTypeQueries.keySet()); diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/searchtypes/OSEventList.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/searchtypes/OSEventList.java index ba36f4def5fb..9dca2a30c410 100644 --- a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/searchtypes/OSEventList.java +++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/searchtypes/OSEventList.java @@ -34,6 +34,7 @@ import org.graylog.shaded.opensearch2.org.opensearch.search.sort.FieldSortBuilder; import org.graylog.shaded.opensearch2.org.opensearch.search.sort.SortOrder; import org.graylog.storage.opensearch2.views.OSGeneratedQueryContext; +import org.graylog2.streams.StreamService; import java.util.List; import java.util.Map; @@ -43,17 +44,21 @@ public class OSEventList implements EventListStrategy { private final ObjectMapper objectMapper; + private final StreamService streamService; @Inject - public OSEventList(ObjectMapper objectMapper) { + public OSEventList(ObjectMapper objectMapper, StreamService streamService) { this.objectMapper = objectMapper; + this.streamService = streamService; } @Override public void doGenerateQueryPart(Query query, EventList eventList, OSGeneratedQueryContext queryContext) { + final Set queryStreamIds = query.usedStreamIds(); + queryStreamIds.addAll(streamService.mapCategoriesToIds(query.usedStreamCategories())); final Set effectiveStreams = eventList.streams().isEmpty() - ? query.usedStreamIds() + ? queryStreamIds : eventList.streams(); final var searchSourceBuilder = queryContext.searchSourceBuilder(eventList); diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/searchtypes/OSMessageList.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/searchtypes/OSMessageList.java index 88573f0f48eb..98cc2f6b7c9e 100644 --- a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/searchtypes/OSMessageList.java +++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/searchtypes/OSMessageList.java @@ -43,6 +43,7 @@ import org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange; import org.graylog2.rest.models.messages.responses.ResultMessageSummary; import org.graylog2.rest.resources.search.responses.SearchResponse; +import org.graylog2.streams.StreamService; import org.joda.time.DateTime; import java.util.ArrayList; @@ -60,14 +61,17 @@ public class OSMessageList implements OSSearchTypeHandler { private final LegacyDecoratorProcessor decoratorProcessor; private final ResultMessageFactory resultMessageFactory; + private final StreamService streamService; private final boolean allowHighlighting; @Inject public OSMessageList(LegacyDecoratorProcessor decoratorProcessor, ResultMessageFactory resultMessageFactory, + StreamService streamService, @Named("allow_highlighting") boolean allowHighlighting) { this.decoratorProcessor = decoratorProcessor; this.resultMessageFactory = resultMessageFactory; + this.streamService = streamService; this.allowHighlighting = allowHighlighting; } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/ViewsBindings.java b/graylog2-server/src/main/java/org/graylog/plugins/views/ViewsBindings.java index f965568e40d7..b5f7862cefae 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/ViewsBindings.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/ViewsBindings.java @@ -56,6 +56,7 @@ import org.graylog.plugins.views.search.filter.AndFilter; import org.graylog.plugins.views.search.filter.OrFilter; import org.graylog.plugins.views.search.filter.QueryStringFilter; +import org.graylog.plugins.views.search.filter.StreamCategoryFilter; import org.graylog.plugins.views.search.filter.StreamFilter; import org.graylog.plugins.views.search.querystrings.LastUsedQueryStringsService; import org.graylog.plugins.views.search.querystrings.MongoLastUsedQueryStringsService; @@ -177,6 +178,7 @@ protected void configure() { registerJacksonSubtype(AndFilter.class); registerJacksonSubtype(OrFilter.class); registerJacksonSubtype(StreamFilter.class); + registerJacksonSubtype(StreamCategoryFilter.class); registerJacksonSubtype(QueryStringFilter.class); // query backends for jackson diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java index 118157ea4523..75a5c8b80530 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java @@ -32,6 +32,7 @@ import org.graylog.plugins.views.search.engine.BackendQuery; import org.graylog.plugins.views.search.engine.EmptyTimeRange; import org.graylog.plugins.views.search.filter.AndFilter; +import org.graylog.plugins.views.search.filter.StreamCategoryFilter; import org.graylog.plugins.views.search.filter.StreamFilter; import org.graylog.plugins.views.search.rest.ExecutionState; import org.graylog.plugins.views.search.rest.ExecutionStateGlobalOverride; @@ -210,6 +211,20 @@ public Set usedStreamIds() { .orElse(Collections.emptySet()); } + @SuppressWarnings("UnstableApiUsage") + public Set usedStreamCategories() { + return Optional.ofNullable(filter()) + .map(optFilter -> { + final Traverser filterTraverser = Traverser.forTree(filter -> firstNonNull(filter.filters(), Collections.emptySet())); + return StreamSupport.stream(filterTraverser.breadthFirst(optFilter).spliterator(), false) + .filter(filter -> filter instanceof StreamCategoryFilter) + .map(streamFilter -> ((StreamCategoryFilter) streamFilter).category()) + .filter(Objects::nonNull) + .collect(toSet()); + }) + .orElse(Collections.emptySet()); + } + public Set streamIdsForPermissionsCheck() { final Set searchTypeStreamIds = searchTypes().stream() .map(SearchType::streams) @@ -219,7 +234,7 @@ public Set streamIdsForPermissionsCheck() { } public boolean hasStreams() { - return !usedStreamIds().isEmpty(); + return !(usedStreamIds().isEmpty() && usedStreamCategories().isEmpty()); } public boolean hasReferencedStreamFilters() { diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/Search.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/Search.java index 22cf40f076f4..2557e1b834b7 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/Search.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/Search.java @@ -168,12 +168,24 @@ public Set usedStreamIds() { return Sets.union(queryStreamIds, searchTypeStreamIds); } + public Set usedStreamCategories() { + return queries().stream() + .map(Query::usedStreamCategories) + .reduce(Collections.emptySet(), Sets::union); + } + public Set streamIdsForPermissionsCheck() { return queries().stream() .map(Query::streamIdsForPermissionsCheck) .reduce(Collections.emptySet(), Sets::union); } + public Set streamCategoriesForPermissionsCheck() { + return queries().stream() + .map(Query::usedStreamCategories) + .reduce(Collections.emptySet(), Sets::union); + } + public Query queryForSearchType(String searchTypeId) { return queries().stream() .filter(q -> q.hasSearchType(searchTypeId)) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/export/CommandFactory.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/export/CommandFactory.java index 0aa4eaabf826..284df43adf9b 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/export/CommandFactory.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/export/CommandFactory.java @@ -27,17 +27,22 @@ import org.graylog2.decorators.Decorator; import org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange; import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; +import org.graylog2.streams.StreamService; import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; public class CommandFactory { private final QueryStringDecorators queryStringDecorator; + private final StreamService streamService; @Inject - public CommandFactory(QueryStringDecorators queryStringDecorator) { + public CommandFactory(QueryStringDecorators queryStringDecorator, + StreamService streamService) { this.queryStringDecorator = queryStringDecorator; + this.streamService = streamService; } public ExportMessagesCommand buildFromRequest(MessagesRequest request) { @@ -56,6 +61,8 @@ public ExportMessagesCommand buildFromRequest(MessagesRequest request) { public ExportMessagesCommand buildWithSearchOnly(Search search, ResultFormat resultFormat) { Query query = queryFrom(search); + final Set queryStreamIds = query.usedStreamIds(); + queryStreamIds.addAll(streamService.mapCategoriesToIds(query.usedStreamCategories())); return builderFrom(resultFormat) .timeRange(resultFormat.timerange().orElse(toAbsolute(query.timerange()))) @@ -65,7 +72,7 @@ public ExportMessagesCommand buildWithSearchOnly(Search search, ResultFormat res .flatMap(List::stream) .collect(Collectors.toList()) ) - .streams(query.usedStreamIds()) + .streams(queryStreamIds) .build(); } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/filter/StreamCategoryFilter.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/filter/StreamCategoryFilter.java new file mode 100644 index 000000000000..feeec05a0122 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/filter/StreamCategoryFilter.java @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.plugins.views.search.filter; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.google.auto.value.AutoValue; +import org.graylog.plugins.views.search.Filter; + +import javax.annotation.Nullable; +import java.util.Set; + +@AutoValue +@JsonTypeName(StreamCategoryFilter.NAME) +@JsonDeserialize(builder = StreamCategoryFilter.Builder.class) +public abstract class StreamCategoryFilter implements Filter { + public static final String NAME = "stream_category"; + + @Override + @JsonProperty + public abstract String type(); + + @Override + @Nullable + @JsonProperty + @JsonInclude(JsonInclude.Include.NON_NULL) + public abstract Set filters(); + + @JsonProperty("category") + public abstract String category(); + + public static Builder builder() { + return Builder.create(); + } + + public abstract Builder toBuilder(); + + @Override + public Filter.Builder toGenericBuilder() { + return toBuilder(); + } + + @AutoValue.Builder + public abstract static class Builder implements Filter.Builder { + @JsonProperty + public abstract Builder type(String type); + + @JsonProperty + public abstract Builder filters(@Nullable Set filters); + + @JsonProperty("category") + public abstract Builder category(String category); + + public abstract StreamCategoryFilter build(); + + @JsonCreator + public static Builder create() { + return new AutoValue_StreamCategoryFilter.Builder().type(NAME); + } + } +} diff --git a/graylog2-server/src/main/java/org/graylog2/plugin/streams/Stream.java b/graylog2-server/src/main/java/org/graylog2/plugin/streams/Stream.java index 39b52c3289b1..6ca9fad5c589 100644 --- a/graylog2-server/src/main/java/org/graylog2/plugin/streams/Stream.java +++ b/graylog2-server/src/main/java/org/graylog2/plugin/streams/Stream.java @@ -89,6 +89,8 @@ public static MatchingType valueOfOrDefault(String name) { String getContentPack(); + List getCategories(); + void setTitle(String title); void setDescription(String description); @@ -99,6 +101,8 @@ public static MatchingType valueOfOrDefault(String name) { void setMatchingType(MatchingType matchingType); + void setCategories(List categories); + Boolean isPaused(); Map asMap(List streamRules); diff --git a/graylog2-server/src/main/java/org/graylog2/streams/StreamDTO.java b/graylog2-server/src/main/java/org/graylog2/streams/StreamDTO.java index c236a98dc5d1..78249f837d73 100644 --- a/graylog2-server/src/main/java/org/graylog2/streams/StreamDTO.java +++ b/graylog2-server/src/main/java/org/graylog2/streams/StreamDTO.java @@ -32,6 +32,7 @@ import javax.annotation.Nullable; import java.util.Collection; import java.util.Date; +import java.util.List; @AutoValue @WithBeanGetter @@ -54,6 +55,7 @@ public abstract class StreamDTO { public static final String FIELD_INDEX_SET_ID = "index_set_id"; public static final String EMBEDDED_ALERT_CONDITIONS = "alert_conditions"; public static final String FIELD_IS_EDITABLE = "is_editable"; + public static final String FIELD_CATEGORIES = "categories"; public static final Stream.MatchingType DEFAULT_MATCHING_TYPE = Stream.MatchingType.AND; @JsonProperty("id") @@ -114,6 +116,10 @@ public abstract class StreamDTO { @JsonProperty(FIELD_IS_EDITABLE) public abstract boolean isEditable(); + @JsonProperty(FIELD_CATEGORIES) + @Nullable + public abstract List categories(); + public abstract Builder toBuilder(); static Builder builder() { @@ -128,7 +134,8 @@ public static Builder create() { .matchingType(DEFAULT_MATCHING_TYPE.toString()) .isDefault(false) .isEditable(false) - .removeMatchesFromDefaultStream(false); + .removeMatchesFromDefaultStream(false) + .categories(List.of()); } @JsonProperty(FIELD_ID) @@ -181,6 +188,9 @@ public static Builder create() { @JsonProperty(FIELD_IS_EDITABLE) public abstract Builder isEditable(boolean isEditable); + @JsonProperty(FIELD_CATEGORIES) + public abstract Builder categories(List categories); + public abstract String id(); public abstract StreamDTO autoBuild(); @@ -206,6 +216,7 @@ public static StreamDTO fromDocument(Document document) { .creatorUserId(document.getString(FIELD_CREATOR_USER_ID)) .indexSetId(document.getString(FIELD_INDEX_SET_ID)) .outputs(document.getList(FIELD_OUTPUTS, ObjectId.class)) + .categories(document.getList(FIELD_CATEGORIES, String.class)) .build(); } } diff --git a/graylog2-server/src/main/java/org/graylog2/streams/StreamImpl.java b/graylog2-server/src/main/java/org/graylog2/streams/StreamImpl.java index 61efaaacf3bd..345b2000cda0 100644 --- a/graylog2-server/src/main/java/org/graylog2/streams/StreamImpl.java +++ b/graylog2-server/src/main/java/org/graylog2/streams/StreamImpl.java @@ -63,6 +63,7 @@ public class StreamImpl extends PersistedImpl implements Stream { public static final String FIELD_DEFAULT_STREAM = "is_default_stream"; public static final String FIELD_REMOVE_MATCHES_FROM_DEFAULT_STREAM = "remove_matches_from_default_stream"; public static final String FIELD_INDEX_SET_ID = "index_set_id"; + public static final String FIELD_CATEGORIES = "categories"; public static final String EMBEDDED_ALERT_CONDITIONS = "alert_conditions"; private final List streamRules; @@ -153,6 +154,17 @@ public void setContentPack(String contentPack) { fields.put(FIELD_CONTENT_PACK, contentPack); } + @Override + @SuppressWarnings("unchecked") + public List getCategories() { + return (List) fields.get(FIELD_CATEGORIES); + } + + @Override + public void setCategories(List categories) { + fields.put(FIELD_CATEGORIES, categories); + } + @Override public Boolean isPaused() { Boolean disabled = getDisabled(); @@ -189,6 +201,7 @@ public Map asMap() { result.put(FIELD_DEFAULT_STREAM, isDefaultStream()); result.put(FIELD_REMOVE_MATCHES_FROM_DEFAULT_STREAM, getRemoveMatchesFromDefaultStream()); result.put(FIELD_INDEX_SET_ID, getIndexSetId()); + result.put(FIELD_CATEGORIES, getCategories()); return result; } diff --git a/graylog2-server/src/main/java/org/graylog2/streams/StreamService.java b/graylog2-server/src/main/java/org/graylog2/streams/StreamService.java index a7121ae9ef6d..ff9cb13bbd0a 100644 --- a/graylog2-server/src/main/java/org/graylog2/streams/StreamService.java +++ b/graylog2-server/src/main/java/org/graylog2/streams/StreamService.java @@ -48,6 +48,8 @@ public interface StreamService extends PersistedService { Set loadByIds(Collection streamIds); + Set mapCategoriesToIds(Collection streamCategories); + Set indexSetIdsByIds(Collection streamIds); List loadAllEnabled(); diff --git a/graylog2-server/src/main/java/org/graylog2/streams/StreamServiceImpl.java b/graylog2-server/src/main/java/org/graylog2/streams/StreamServiceImpl.java index 79a626959529..986fa601bb74 100644 --- a/graylog2-server/src/main/java/org/graylog2/streams/StreamServiceImpl.java +++ b/graylog2-server/src/main/java/org/graylog2/streams/StreamServiceImpl.java @@ -70,6 +70,7 @@ import static com.mongodb.client.model.Projections.excludeId; import static com.mongodb.client.model.Projections.fields; import static com.mongodb.client.model.Projections.include; +import static org.graylog2.streams.StreamImpl.FIELD_ID; import static org.graylog2.streams.StreamImpl.FIELD_INDEX_SET_ID; import static org.graylog2.streams.StreamImpl.FIELD_TITLE; @@ -282,6 +283,16 @@ public Set loadByIds(Collection streamIds) { return ImmutableSet.copyOf(loadAll(query)); } + @Override + public Set mapCategoriesToIds(Collection categories) { + final DBObject query = QueryBuilder.start(StreamImpl.FIELD_CATEGORIES).in(categories).get(); + final DBObject onlyIdField = DBProjection.include(FIELD_ID); + try (var cursor = collection(StreamImpl.class).find(query, onlyIdField); + var stream = StreamSupport.stream(cursor.spliterator(), false)) { + return stream.map(s -> s.get(FIELD_ID).toString()).collect(Collectors.toSet()); + } + } + @Override public Set indexSetIdsByIds(Collection streamIds) { Set dataStreamIds = streamIds.stream() From c272deaaa47812766ed5fdc399231fe1bf289c4c Mon Sep 17 00:00:00 2001 From: Zack King Date: Wed, 7 Aug 2024 15:16:07 -0500 Subject: [PATCH 02/13] Fix introduced test failures --- .../search/export/CommandFactoryTest.java | 38 ++++++++++++++++++- .../java/org/graylog2/streams/StreamMock.java | 12 ++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/graylog2-server/src/test/java/org/graylog/plugins/views/search/export/CommandFactoryTest.java b/graylog2-server/src/test/java/org/graylog/plugins/views/search/export/CommandFactoryTest.java index 9cb85b944e05..2ec0c7746e25 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/views/search/export/CommandFactoryTest.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/views/search/export/CommandFactoryTest.java @@ -27,17 +27,30 @@ import org.graylog.plugins.views.search.searchfilters.model.InlineQueryStringSearchFilter; import org.graylog.plugins.views.search.searchtypes.MessageList; import org.graylog.plugins.views.search.searchtypes.pivot.Pivot; +import org.graylog.security.entities.EntityOwnershipService; +import org.graylog.testing.mongodb.MongoDBInstance; import org.graylog2.decorators.Decorator; +import org.graylog2.events.ClusterEventBus; +import org.graylog2.indexer.MongoIndexSet; +import org.graylog2.indexer.indexset.IndexSetService; import org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange; import org.graylog2.plugin.indexer.searches.timeranges.InvalidRangeParametersException; import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange; import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; +import org.graylog2.streams.OutputService; +import org.graylog2.streams.StreamRuleService; +import org.graylog2.streams.StreamServiceImpl; +import org.junit.Rule; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; import java.util.Arrays; import java.util.List; import java.util.Optional; +import java.util.Set; import static com.google.common.collect.Lists.newArrayList; import static org.assertj.core.api.Assertions.assertThat; @@ -51,13 +64,34 @@ import static org.mockito.Mockito.mock; class CommandFactoryTest { + @Rule + public final MongoDBInstance mongodb = MongoDBInstance.createForClass(); + + @Rule + public final MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Mock + private StreamRuleService streamRuleService; + @Mock + private OutputService outputService; + @Mock + private IndexSetService indexSetService; + @Mock + private MongoIndexSet.Factory factory; + @Mock + private EntityOwnershipService entityOwnershipService; + + private StreamServiceImpl mockStreamService; private CommandFactory sut; @BeforeEach void setUp() { final QueryStringDecorators emptyDecorator = new QueryStringDecorators(Optional.empty()); - sut = new CommandFactory(emptyDecorator); + mongodb.start(); + this.mockStreamService = new StreamServiceImpl(mongodb.mongoConnection(), streamRuleService, + outputService, indexSetService, factory, entityOwnershipService, new ClusterEventBus(), Set.of()); + sut = new CommandFactory(emptyDecorator, mockStreamService); } @Test @@ -355,7 +389,7 @@ void appliesQueryDecorators() { } })); - ExportMessagesCommand command = new CommandFactory(decorators).buildWithSearchOnly(s, ResultFormat.builder().build()); + ExportMessagesCommand command = new CommandFactory(decorators, mockStreamService).buildWithSearchOnly(s, ResultFormat.builder().build()); assertThat(command.queryString()).isEqualTo(ElasticsearchQueryString.of("decorated")); } diff --git a/graylog2-server/src/test/java/org/graylog2/streams/StreamMock.java b/graylog2-server/src/test/java/org/graylog2/streams/StreamMock.java index 8515f4100cc8..f004f030c4eb 100644 --- a/graylog2-server/src/test/java/org/graylog2/streams/StreamMock.java +++ b/graylog2-server/src/test/java/org/graylog2/streams/StreamMock.java @@ -48,6 +48,7 @@ public class StreamMock implements Stream { private boolean disabled; private String contentPack; private List streamRules; + private List categories; private MatchingType matchingType; private boolean defaultStream; private boolean removeMatchesFromDefaultStream; @@ -69,6 +70,7 @@ public StreamMock(Map stream, List streamRules) { this.matchingType = (MatchingType) stream.getOrDefault(StreamImpl.FIELD_MATCHING_TYPE, MatchingType.AND); this.defaultStream = (boolean) stream.getOrDefault(StreamImpl.FIELD_DEFAULT_STREAM, false); this.removeMatchesFromDefaultStream = (boolean) stream.getOrDefault(StreamImpl.FIELD_REMOVE_MATCHES_FROM_DEFAULT_STREAM, false); + this.categories = (List) stream.getOrDefault(StreamImpl.FIELD_CATEGORIES, List.of()); this.indexSet = new TestIndexSet(IndexSetConfig.create( "index-set-id", "title", @@ -134,6 +136,11 @@ public String getContentPack() { return contentPack; } + @Override + public List getCategories() { + return categories; + } + @Override public void setTitle(String title) { this.title = title; @@ -189,6 +196,11 @@ public void setMatchingType(MatchingType matchingType) { this.matchingType = matchingType; } + @Override + public void setCategories(List categories) { + this.categories = categories; + } + @Override public boolean isDefaultStream() { return defaultStream; From 2ac60fa218233a12e4cfba10ab03dfa36d96bf49 Mon Sep 17 00:00:00 2001 From: Zack King Date: Thu, 8 Aug 2024 08:34:41 -0500 Subject: [PATCH 03/13] Remove streamService from eventlists and messagelists --- .../elasticsearch7/views/searchtypes/ESEventList.java | 9 ++------- .../elasticsearch7/views/searchtypes/ESMessageList.java | 4 ---- .../opensearch2/views/searchtypes/OSEventList.java | 9 ++------- .../opensearch2/views/searchtypes/OSMessageList.java | 4 ---- 4 files changed, 4 insertions(+), 22 deletions(-) diff --git a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/searchtypes/ESEventList.java b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/searchtypes/ESEventList.java index 8baf10c604d2..6daf563c5a65 100644 --- a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/searchtypes/ESEventList.java +++ b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/searchtypes/ESEventList.java @@ -34,7 +34,6 @@ import org.graylog.shaded.elasticsearch7.org.elasticsearch.search.sort.FieldSortBuilder; import org.graylog.shaded.elasticsearch7.org.elasticsearch.search.sort.SortOrder; import org.graylog.storage.elasticsearch7.views.ESGeneratedQueryContext; -import org.graylog2.streams.StreamService; import java.util.List; import java.util.Map; @@ -44,21 +43,17 @@ public class ESEventList implements ESSearchTypeHandler { private final ObjectMapper objectMapper; - private final StreamService streamService; @Inject - public ESEventList(ObjectMapper objectMapper, StreamService streamService) { + public ESEventList(ObjectMapper objectMapper) { this.objectMapper = objectMapper; - this.streamService = streamService; } @Override public void doGenerateQueryPart(Query query, EventList eventList, ESGeneratedQueryContext queryContext) { - final Set queryStreamIds = query.usedStreamIds(); - queryStreamIds.addAll(streamService.mapCategoriesToIds(query.usedStreamCategories())); final Set effectiveStreams = eventList.streams().isEmpty() - ? queryStreamIds + ? query.usedStreamIds() : eventList.streams(); final var searchSourceBuilder = queryContext.searchSourceBuilder(eventList); diff --git a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/searchtypes/ESMessageList.java b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/searchtypes/ESMessageList.java index 0ff0aa9eec65..c12de5dc6781 100644 --- a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/searchtypes/ESMessageList.java +++ b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/searchtypes/ESMessageList.java @@ -43,7 +43,6 @@ import org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange; import org.graylog2.rest.models.messages.responses.ResultMessageSummary; import org.graylog2.rest.resources.search.responses.SearchResponse; -import org.graylog2.streams.StreamService; import org.joda.time.DateTime; import java.util.ArrayList; @@ -61,17 +60,14 @@ public class ESMessageList implements ESSearchTypeHandler { private final LegacyDecoratorProcessor decoratorProcessor; private final ResultMessageFactory resultMessageFactory; - private final StreamService streamService; private final boolean allowHighlighting; @Inject public ESMessageList(LegacyDecoratorProcessor decoratorProcessor, ResultMessageFactory resultMessageFactory, - StreamService streamService, @Named("allow_highlighting") boolean allowHighlighting) { this.decoratorProcessor = decoratorProcessor; this.resultMessageFactory = resultMessageFactory; - this.streamService = streamService; this.allowHighlighting = allowHighlighting; } diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/searchtypes/OSEventList.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/searchtypes/OSEventList.java index 9dca2a30c410..ba36f4def5fb 100644 --- a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/searchtypes/OSEventList.java +++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/searchtypes/OSEventList.java @@ -34,7 +34,6 @@ import org.graylog.shaded.opensearch2.org.opensearch.search.sort.FieldSortBuilder; import org.graylog.shaded.opensearch2.org.opensearch.search.sort.SortOrder; import org.graylog.storage.opensearch2.views.OSGeneratedQueryContext; -import org.graylog2.streams.StreamService; import java.util.List; import java.util.Map; @@ -44,21 +43,17 @@ public class OSEventList implements EventListStrategy { private final ObjectMapper objectMapper; - private final StreamService streamService; @Inject - public OSEventList(ObjectMapper objectMapper, StreamService streamService) { + public OSEventList(ObjectMapper objectMapper) { this.objectMapper = objectMapper; - this.streamService = streamService; } @Override public void doGenerateQueryPart(Query query, EventList eventList, OSGeneratedQueryContext queryContext) { - final Set queryStreamIds = query.usedStreamIds(); - queryStreamIds.addAll(streamService.mapCategoriesToIds(query.usedStreamCategories())); final Set effectiveStreams = eventList.streams().isEmpty() - ? queryStreamIds + ? query.usedStreamIds() : eventList.streams(); final var searchSourceBuilder = queryContext.searchSourceBuilder(eventList); diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/searchtypes/OSMessageList.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/searchtypes/OSMessageList.java index 98cc2f6b7c9e..88573f0f48eb 100644 --- a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/searchtypes/OSMessageList.java +++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/searchtypes/OSMessageList.java @@ -43,7 +43,6 @@ import org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange; import org.graylog2.rest.models.messages.responses.ResultMessageSummary; import org.graylog2.rest.resources.search.responses.SearchResponse; -import org.graylog2.streams.StreamService; import org.joda.time.DateTime; import java.util.ArrayList; @@ -61,17 +60,14 @@ public class OSMessageList implements OSSearchTypeHandler { private final LegacyDecoratorProcessor decoratorProcessor; private final ResultMessageFactory resultMessageFactory; - private final StreamService streamService; private final boolean allowHighlighting; @Inject public OSMessageList(LegacyDecoratorProcessor decoratorProcessor, ResultMessageFactory resultMessageFactory, - StreamService streamService, @Named("allow_highlighting") boolean allowHighlighting) { this.decoratorProcessor = decoratorProcessor; this.resultMessageFactory = resultMessageFactory; - this.streamService = streamService; this.allowHighlighting = allowHighlighting; } From 3c0d90db4993a4e59c167686147cbf81e183a98d Mon Sep 17 00:00:00 2001 From: Zack King Date: Tue, 13 Aug 2024 10:53:27 -0500 Subject: [PATCH 04/13] Move StreamCategory resolution to SearchExecutor from SearchBackend --- .../views/ElasticsearchBackend.java | 10 ++----- .../opensearch2/views/OpenSearchBackend.java | 9 +----- .../graylog/plugins/views/search/Search.java | 30 +++++++++++++++++++ .../PluggableSearchNormalization.java | 18 ++++++++--- .../search/engine/SearchExecutorTest.java | 6 +++- .../rest/SearchResourceExecutionTest.java | 8 +++-- 6 files changed, 57 insertions(+), 24 deletions(-) diff --git a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackend.java b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackend.java index 3d3a477ee102..ef7db7cc91c5 100644 --- a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackend.java +++ b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackend.java @@ -59,7 +59,6 @@ import org.graylog2.indexer.ranges.IndexRange; import org.graylog2.plugin.Message; import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; -import org.graylog2.streams.StreamService; import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -88,7 +87,6 @@ public class ElasticsearchBackend implements QueryBackend executionStatsCollector; - private final StreamService streamService; @Inject public ElasticsearchBackend(Map>> elasticsearchSearchTypeHandlers, @@ -97,7 +95,6 @@ public ElasticsearchBackend(Map executionStatsCollector, - StreamService streamService, @Named("allow_leading_wildcard_searches") boolean allowLeadingWildcard) { this.elasticsearchSearchTypeHandlers = elasticsearchSearchTypeHandlers; this.client = client; @@ -106,7 +103,6 @@ public ElasticsearchBackend(Map validation final SearchSourceBuilder searchTypeSourceBuilder = queryContext.searchSourceBuilder(searchType); final Set effectiveStreamIds = query.effectiveStreams(searchType); - effectiveStreamIds.addAll(streamService.mapCategoriesToIds(query.usedStreamCategories())); + final BoolQueryBuilder searchTypeOverrides = QueryBuilders.boolQuery() .must(searchTypeSourceBuilder.query()) .must( @@ -242,9 +238,7 @@ public QueryResult doRun(SearchJob job, Query query, ESGeneratedQueryContext que LOG.debug("Running query {} for job {}", query.id(), job.getId()); final HashMap resultsMap = Maps.newHashMap(); - final Set usedStreams = query.usedStreamIds(); - usedStreams.addAll(streamService.mapCategoriesToIds(query.usedStreamCategories())); - final Set affectedIndices = indexLookup.indexNamesForStreamsInTimeRange(usedStreams, query.timerange()); + final Set affectedIndices = indexLookup.indexNamesForStreamsInTimeRange(query.usedStreamIds(), query.timerange()); final Map searchTypeQueries = queryContext.searchTypeQueries(); final List searchTypeIds = new ArrayList<>(searchTypeQueries.keySet()); diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/OpenSearchBackend.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/OpenSearchBackend.java index 8d6d553da127..35ad23e1fe82 100644 --- a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/OpenSearchBackend.java +++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/OpenSearchBackend.java @@ -61,7 +61,6 @@ import org.graylog2.plugin.Message; import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; import org.graylog2.plugin.streams.Stream; -import org.graylog2.streams.StreamService; import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -89,7 +88,6 @@ public class OpenSearchBackend implements QueryBackend private final UsedSearchFiltersToQueryStringsMapper usedSearchFiltersToQueryStringsMapper; private final boolean allowLeadingWildcard; private final StatsCollector executionStatsCollector; - private final StreamService streamService; @Inject public OpenSearchBackend(Map>> elasticsearchSearchTypeHandlers, @@ -98,7 +96,6 @@ public OpenSearchBackend(Map executionStatsCollector, - StreamService streamService, @Named("allow_leading_wildcard_searches") boolean allowLeadingWildcard) { this.openSearchSearchTypeHandlers = elasticsearchSearchTypeHandlers; this.client = client; @@ -107,7 +104,6 @@ public OpenSearchBackend(Map validation final SearchSourceBuilder searchTypeSourceBuilder = queryContext.searchSourceBuilder(searchType); final Set effectiveStreamIds = query.effectiveStreams(searchType); - effectiveStreamIds.addAll(streamService.mapCategoriesToIds(query.usedStreamCategories())); final BoolQueryBuilder searchTypeOverrides = QueryBuilders.boolQuery() .must(searchTypeSourceBuilder.query()) .must( @@ -245,9 +240,7 @@ public QueryResult doRun(SearchJob job, Query query, OSGeneratedQueryContext que LOG.debug("Running query {} for job {}", query.id(), job.getId()); final HashMap resultsMap = Maps.newHashMap(); - final Set usedStreams = query.usedStreamIds(); - usedStreams.addAll(streamService.mapCategoriesToIds(query.usedStreamCategories())); - final Set affectedIndices = indexLookup.indexNamesForStreamsInTimeRange(usedStreams, query.timerange()); + final Set affectedIndices = indexLookup.indexNamesForStreamsInTimeRange(query.usedStreamIds(), query.timerange()); final Map searchTypeQueries = queryContext.searchTypeQueries(); final List searchTypeIds = new ArrayList<>(searchTypeQueries.keySet()); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/Search.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/Search.java index 2557e1b834b7..76eb01a42d95 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/Search.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/Search.java @@ -28,6 +28,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.graph.MutableGraph; +import org.graylog.plugins.views.search.permissions.StreamPermissions; import org.graylog.plugins.views.search.rest.ExecutionState; import org.graylog.plugins.views.search.views.PluginMetadataSummary; import org.graylog2.contentpacks.ContentPackable; @@ -43,13 +44,17 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static java.util.stream.Collectors.toSet; @@ -146,10 +151,35 @@ public Search addStreamsToQueriesWithoutStreams(Supplier> defaultStr return toBuilder().queries(newQueries).build(); } + public Search addStreamsToQueriesWithCategories(Function, Stream> categoryMappingFunction, + StreamPermissions streamPermissions) { + if (!hasQueriesWithStreamCategories()) { + return this; + } + final Set withStreamCategories = queries().stream().filter(q -> !q.usedStreamCategories().isEmpty()).collect(toSet()); + final Set withoutStreamCategories = Sets.difference(queries(), withStreamCategories); + final Set withMappedStreamCategories = new HashSet<>(); + + for (Query query : withStreamCategories) { + final Set mappedStreamIds = categoryMappingFunction.apply(query.usedStreamCategories()) + .filter(streamPermissions::canReadStream) + .collect(toSet()); + withMappedStreamCategories.add(query.addStreamsToFilter(mappedStreamIds)); + } + + final ImmutableSet newQueries = Sets.union(withMappedStreamCategories, withoutStreamCategories).immutableCopy(); + + return toBuilder().queries(newQueries).build(); + } + private boolean hasQueriesWithoutStreams() { return !queries().stream().allMatch(Query::hasStreams); } + private boolean hasQueriesWithStreamCategories() { + return queries().stream().anyMatch(q -> !q.usedStreamCategories().isEmpty()); + } + public abstract Builder toBuilder(); public static Builder builder() { diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/normalization/PluggableSearchNormalization.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/normalization/PluggableSearchNormalization.java index 0ca114b82ec7..6b5043a8a100 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/normalization/PluggableSearchNormalization.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/normalization/PluggableSearchNormalization.java @@ -24,27 +24,34 @@ import org.graylog.plugins.views.search.rest.ExecutionState; import org.graylog.plugins.views.search.rest.ExecutionStateGlobalOverride; import org.graylog2.plugin.Tools; +import org.graylog2.streams.StreamService; import org.joda.time.DateTime; +import java.util.Collection; import java.util.Collections; import java.util.Optional; import java.util.Set; +import java.util.function.Function; +import java.util.stream.Stream; import static com.google.common.base.MoreObjects.firstNonNull; public class PluggableSearchNormalization implements SearchNormalization { private final Set pluggableNormalizers; private final Set postValidationNormalizers; + private final Function, Stream> streamCategoryMapper; @Inject public PluggableSearchNormalization(Set pluggableNormalizers, - @PostValidation Set postValidationNormalizers) { + @PostValidation Set postValidationNormalizers, + StreamService streamService) { this.pluggableNormalizers = pluggableNormalizers; this.postValidationNormalizers = postValidationNormalizers; + this.streamCategoryMapper = (categories) -> streamService.mapCategoriesToIds(categories).stream(); } - public PluggableSearchNormalization(Set pluggableNormalizers) { - this(pluggableNormalizers, Collections.emptySet()); + public PluggableSearchNormalization(Set pluggableNormalizers, StreamService streamService) { + this(pluggableNormalizers, Collections.emptySet(), streamService); } private Search normalize(Search search, Set normalizers) { @@ -68,7 +75,9 @@ private Query normalize(final Query query, @Override public Search preValidation(Search search, SearchUser searchUser, ExecutionState executionState) { - final Search searchWithStreams = search.addStreamsToQueriesWithoutStreams(() -> searchUser.streams().loadMessageStreamsWithFallback()); + final Search searchWithStreams = search + .addStreamsToQueriesWithoutStreams(() -> searchUser.streams().loadMessageStreamsWithFallback()) + .addStreamsToQueriesWithCategories(streamCategoryMapper, searchUser); final var now = referenceDateFromOverrideOrNow(executionState); final var normalizedSearch = searchWithStreams.applyExecutionState(firstNonNull(executionState, ExecutionState.empty())) .withReferenceDate(now); @@ -93,6 +102,7 @@ public Query preValidation(final Query query, final ParameterProvider parameterP Query normalizedQuery = query; if (!query.hasStreams()) { normalizedQuery = query.addStreamsToFilter(searchUser.streams().loadMessageStreamsWithFallback()); + } else if (!query.usedStreamCategories().isEmpty()) { } if (!executionState.equals(ExecutionState.empty())) { diff --git a/graylog2-server/src/test/java/org/graylog/plugins/views/search/engine/SearchExecutorTest.java b/graylog2-server/src/test/java/org/graylog/plugins/views/search/engine/SearchExecutorTest.java index fe56cd4e7ec9..0f1a7c912449 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/views/search/engine/SearchExecutorTest.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/views/search/engine/SearchExecutorTest.java @@ -42,6 +42,7 @@ import org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange; import org.graylog2.plugin.system.NodeId; import org.graylog2.shared.rest.exceptions.MissingStreamPermissionException; +import org.graylog2.streams.StreamService; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -78,6 +79,9 @@ public class SearchExecutorTest { @Mock private NodeId nodeId; + @Mock + private StreamService streamService; + @Captor private ArgumentCaptor searchJobCaptor; @@ -97,7 +101,7 @@ void setUp() { Optional.of((queryString, job, query) -> PositionTrackingQuery.of("decorated")) ) ) - ))); + ), streamService)); when(queryEngine.execute(any(), any(), any())).thenAnswer(invocation -> { final SearchJob searchJob = invocation.getArgument(0); searchJob.addQueryResultFuture("query", CompletableFuture.completedFuture(QueryResult.emptyResult())); diff --git a/graylog2-server/src/test/java/org/graylog/plugins/views/search/rest/SearchResourceExecutionTest.java b/graylog2-server/src/test/java/org/graylog/plugins/views/search/rest/SearchResourceExecutionTest.java index 569f5435c074..dfdbf9c3327c 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/views/search/rest/SearchResourceExecutionTest.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/views/search/rest/SearchResourceExecutionTest.java @@ -40,6 +40,7 @@ import org.graylog2.plugin.system.NodeId; import org.graylog2.plugin.system.SimpleNodeId; import org.graylog2.shared.rest.exceptions.MissingStreamPermissionException; +import org.graylog2.streams.StreamService; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -58,9 +59,7 @@ import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -90,6 +89,9 @@ public class SearchResourceExecutionTest { @Mock private ClusterConfigService clusterConfigService; + @Mock + private StreamService streamService; + private final NodeId nodeId = new SimpleNodeId("5ca1ab1e-0000-4000-a000-000000000000"); private SearchResource searchResource; @@ -103,7 +105,7 @@ public void setUp() { searchJobService, queryEngine, new PluggableSearchValidation(executionGuard, Collections.emptySet()), - new PluggableSearchNormalization(Collections.emptySet())); + new PluggableSearchNormalization(Collections.emptySet(), streamService)); this.searchResource = new SearchResource(searchDomain, searchExecutor, searchJobService, eventBus, clusterConfigService) { @Override From ddec6c0cfe492927e9bb80063062cd1eb573ff9e Mon Sep 17 00:00:00 2001 From: Zack King Date: Tue, 13 Aug 2024 12:16:56 -0500 Subject: [PATCH 05/13] revert unnecessary changes --- .../storage/elasticsearch7/views/ElasticsearchBackend.java | 1 + .../graylog/storage/opensearch2/views/OpenSearchBackend.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackend.java b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackend.java index ef7db7cc91c5..9f549d13a74f 100644 --- a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackend.java +++ b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackend.java @@ -76,6 +76,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; public class ElasticsearchBackend implements QueryBackend { private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchBackend.class); diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/OpenSearchBackend.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/OpenSearchBackend.java index 35ad23e1fe82..78fe70799a93 100644 --- a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/OpenSearchBackend.java +++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/OpenSearchBackend.java @@ -77,6 +77,7 @@ import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; public class OpenSearchBackend implements QueryBackend { private static final Logger LOG = LoggerFactory.getLogger(OpenSearchBackend.class); @@ -158,6 +159,7 @@ public OSGeneratedQueryContext generate(Query query, Set validation final SearchSourceBuilder searchTypeSourceBuilder = queryContext.searchSourceBuilder(searchType); final Set effectiveStreamIds = query.effectiveStreams(searchType); + final BoolQueryBuilder searchTypeOverrides = QueryBuilders.boolQuery() .must(searchTypeSourceBuilder.query()) .must( From 5f19b8552891a37cbcbe7d0ada94cae8fa902dd1 Mon Sep 17 00:00:00 2001 From: Zack King Date: Tue, 13 Aug 2024 12:51:27 -0500 Subject: [PATCH 06/13] Add logic to populate queries with streamcategories with streamIds --- .../org/graylog/plugins/views/search/Query.java | 13 +++++++++++++ .../normalization/PluggableSearchNormalization.java | 1 + 2 files changed, 14 insertions(+) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java index 75a5c8b80530..6c82ac6b5466 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java @@ -34,6 +34,7 @@ import org.graylog.plugins.views.search.filter.AndFilter; import org.graylog.plugins.views.search.filter.StreamCategoryFilter; import org.graylog.plugins.views.search.filter.StreamFilter; +import org.graylog.plugins.views.search.permissions.StreamPermissions; import org.graylog.plugins.views.search.rest.ExecutionState; import org.graylog.plugins.views.search.rest.ExecutionStateGlobalOverride; import org.graylog.plugins.views.search.rest.SearchTypeExecutionState; @@ -51,13 +52,16 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import java.util.stream.StreamSupport; import static com.google.common.base.MoreObjects.firstNonNull; @@ -246,6 +250,15 @@ public Query addStreamsToFilter(Set streamIds) { return toBuilder().filter(newFilter).build(); } + public Query mapStreamCategoriesToIds(Function, Stream> categoryMappingFunction, + StreamPermissions streamPermissions) { + Set streamIds = categoryMappingFunction.apply(usedStreamCategories()) + .filter(streamPermissions::canReadStream) + .collect(toSet()); + final Filter newFilter = addStreamsTo(filter(), streamIds); + return toBuilder().filter(newFilter).build(); + } + private Filter addStreamsTo(Filter filter, Set streamIds) { final Filter streamIdFilter = StreamFilter.anyIdOf(streamIds.toArray(new String[]{})); if (filter == null) { diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/normalization/PluggableSearchNormalization.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/normalization/PluggableSearchNormalization.java index 6b5043a8a100..ba92911e6ad9 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/normalization/PluggableSearchNormalization.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/normalization/PluggableSearchNormalization.java @@ -103,6 +103,7 @@ public Query preValidation(final Query query, final ParameterProvider parameterP if (!query.hasStreams()) { normalizedQuery = query.addStreamsToFilter(searchUser.streams().loadMessageStreamsWithFallback()); } else if (!query.usedStreamCategories().isEmpty()) { + normalizedQuery = query.mapStreamCategoriesToIds(streamCategoryMapper, searchUser); } if (!executionState.equals(ExecutionState.empty())) { From 38390da06043f48768dee2b1ddc823b782183a79 Mon Sep 17 00:00:00 2001 From: Zack King Date: Tue, 13 Aug 2024 13:51:11 -0500 Subject: [PATCH 07/13] Move streamcategory mapping from CommandFactory to MessagesResource --- .../graylog/plugins/views/search/Query.java | 3 +- .../views/search/export/CommandFactory.java | 7 +--- .../views/search/rest/MessagesResource.java | 34 +++++++++-------- .../search/export/CommandFactoryTest.java | 38 +------------------ 4 files changed, 23 insertions(+), 59 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java index 6c82ac6b5466..7bd1dac7126b 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java @@ -255,8 +255,7 @@ public Query mapStreamCategoriesToIds(Function, Stream streamIds = categoryMappingFunction.apply(usedStreamCategories()) .filter(streamPermissions::canReadStream) .collect(toSet()); - final Filter newFilter = addStreamsTo(filter(), streamIds); - return toBuilder().filter(newFilter).build(); + return addStreamsToFilter(streamIds); } private Filter addStreamsTo(Filter filter, Set streamIds) { diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/export/CommandFactory.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/export/CommandFactory.java index 284df43adf9b..aa97ff3a69fc 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/export/CommandFactory.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/export/CommandFactory.java @@ -27,7 +27,6 @@ import org.graylog2.decorators.Decorator; import org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange; import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; -import org.graylog2.streams.StreamService; import java.util.Collections; import java.util.List; @@ -36,13 +35,10 @@ public class CommandFactory { private final QueryStringDecorators queryStringDecorator; - private final StreamService streamService; @Inject - public CommandFactory(QueryStringDecorators queryStringDecorator, - StreamService streamService) { + public CommandFactory(QueryStringDecorators queryStringDecorator) { this.queryStringDecorator = queryStringDecorator; - this.streamService = streamService; } public ExportMessagesCommand buildFromRequest(MessagesRequest request) { @@ -62,7 +58,6 @@ public ExportMessagesCommand buildFromRequest(MessagesRequest request) { public ExportMessagesCommand buildWithSearchOnly(Search search, ResultFormat resultFormat) { Query query = queryFrom(search); final Set queryStreamIds = query.usedStreamIds(); - queryStreamIds.addAll(streamService.mapCategoriesToIds(query.usedStreamCategories())); return builderFrom(resultFormat) .timeRange(resultFormat.timerange().orElse(toAbsolute(query.timerange()))) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/rest/MessagesResource.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/rest/MessagesResource.java index 7547d5de6220..743217721696 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/rest/MessagesResource.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/rest/MessagesResource.java @@ -20,6 +20,16 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; +import jakarta.inject.Inject; +import jakarta.validation.Valid; +import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.NotFoundException; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.Context; import org.apache.shiro.authz.annotation.RequiresAuthentication; import org.glassfish.jersey.server.ChunkedOutput; import org.graylog.plugins.views.search.Search; @@ -49,25 +59,15 @@ import org.graylog2.plugin.rest.PluginRestResource; import org.graylog2.rest.MoreMediaTypes; import org.graylog2.shared.rest.resources.RestResource; +import org.graylog2.streams.StreamService; import org.joda.time.DateTimeZone; -import jakarta.inject.Inject; - -import jakarta.validation.Valid; - -import jakarta.ws.rs.BadRequestException; -import jakarta.ws.rs.GET; -import jakarta.ws.rs.NotFoundException; -import jakarta.ws.rs.POST; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.PathParam; -import jakarta.ws.rs.Produces; -import jakarta.ws.rs.core.Context; - +import java.util.Collection; import java.util.Objects; import java.util.Optional; import java.util.function.Consumer; import java.util.function.Function; +import java.util.stream.Stream; import static org.graylog2.shared.rest.documentation.generator.Generator.CLOUD_VISIBLE; @@ -85,6 +85,7 @@ public class MessagesResource extends RestResource implements PluginRestResource //allow mocking Function>, ChunkedOutput> asyncRunner = ChunkedRunner::runAsync; Function messagesExporterFactory; + Function, Stream> streamCategoryMapper; @Inject public MessagesResource( @@ -93,13 +94,16 @@ public MessagesResource( SearchDomain searchDomain, SearchExecutionGuard executionGuard, @SuppressWarnings("UnstableApiUsage") EventBus eventBus, - ExportJobService exportJobService, QueryValidationService queryValidationService) { + ExportJobService exportJobService, + QueryValidationService queryValidationService, + StreamService streamService) { this.commandFactory = commandFactory; this.searchDomain = searchDomain; this.executionGuard = executionGuard; this.exportJobService = exportJobService; this.queryValidationService = queryValidationService; this.messagesExporterFactory = context -> new AuditingMessagesExporter(context, eventBus, exporter); + this.streamCategoryMapper = categories -> streamService.mapCategoriesToIds(categories).stream(); } @ApiOperation( @@ -246,7 +250,7 @@ private Search loadSearch(String searchId, ExecutionState executionState, Search .orElseThrow(() -> new NotFoundException("Search with id " + searchId + " does not exist")); search = search.addStreamsToQueriesWithoutStreams(() -> searchUser.streams().loadMessageStreamsWithFallback()); - + search = search.addStreamsToQueriesWithCategories(streamCategoryMapper, searchUser); search = search.applyExecutionState(executionState); executionGuard.check(search, searchUser::canReadStream); diff --git a/graylog2-server/src/test/java/org/graylog/plugins/views/search/export/CommandFactoryTest.java b/graylog2-server/src/test/java/org/graylog/plugins/views/search/export/CommandFactoryTest.java index 2ec0c7746e25..9cb85b944e05 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/views/search/export/CommandFactoryTest.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/views/search/export/CommandFactoryTest.java @@ -27,30 +27,17 @@ import org.graylog.plugins.views.search.searchfilters.model.InlineQueryStringSearchFilter; import org.graylog.plugins.views.search.searchtypes.MessageList; import org.graylog.plugins.views.search.searchtypes.pivot.Pivot; -import org.graylog.security.entities.EntityOwnershipService; -import org.graylog.testing.mongodb.MongoDBInstance; import org.graylog2.decorators.Decorator; -import org.graylog2.events.ClusterEventBus; -import org.graylog2.indexer.MongoIndexSet; -import org.graylog2.indexer.indexset.IndexSetService; import org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange; import org.graylog2.plugin.indexer.searches.timeranges.InvalidRangeParametersException; import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange; import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; -import org.graylog2.streams.OutputService; -import org.graylog2.streams.StreamRuleService; -import org.graylog2.streams.StreamServiceImpl; -import org.junit.Rule; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; import java.util.Arrays; import java.util.List; import java.util.Optional; -import java.util.Set; import static com.google.common.collect.Lists.newArrayList; import static org.assertj.core.api.Assertions.assertThat; @@ -64,34 +51,13 @@ import static org.mockito.Mockito.mock; class CommandFactoryTest { - @Rule - public final MongoDBInstance mongodb = MongoDBInstance.createForClass(); - - @Rule - public final MockitoRule mockitoRule = MockitoJUnit.rule(); - - @Mock - private StreamRuleService streamRuleService; - @Mock - private OutputService outputService; - @Mock - private IndexSetService indexSetService; - @Mock - private MongoIndexSet.Factory factory; - @Mock - private EntityOwnershipService entityOwnershipService; - - private StreamServiceImpl mockStreamService; private CommandFactory sut; @BeforeEach void setUp() { final QueryStringDecorators emptyDecorator = new QueryStringDecorators(Optional.empty()); - mongodb.start(); - this.mockStreamService = new StreamServiceImpl(mongodb.mongoConnection(), streamRuleService, - outputService, indexSetService, factory, entityOwnershipService, new ClusterEventBus(), Set.of()); - sut = new CommandFactory(emptyDecorator, mockStreamService); + sut = new CommandFactory(emptyDecorator); } @Test @@ -389,7 +355,7 @@ void appliesQueryDecorators() { } })); - ExportMessagesCommand command = new CommandFactory(decorators, mockStreamService).buildWithSearchOnly(s, ResultFormat.builder().build()); + ExportMessagesCommand command = new CommandFactory(decorators).buildWithSearchOnly(s, ResultFormat.builder().build()); assertThat(command.queryString()).isEqualTo(ElasticsearchQueryString.of("decorated")); } From 2175a6f09ddd9a9a8180257963c6b849ce004f3f Mon Sep 17 00:00:00 2001 From: Zack King Date: Tue, 13 Aug 2024 14:20:47 -0500 Subject: [PATCH 08/13] Fix MessagesResourceTest --- .../views/search/rest/MessagesResource.java | 15 +++++++++++++-- .../views/search/rest/MessagesResourceTest.java | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/rest/MessagesResource.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/rest/MessagesResource.java index 743217721696..3ac00be40065 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/rest/MessagesResource.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/rest/MessagesResource.java @@ -81,11 +81,11 @@ public class MessagesResource extends RestResource implements PluginRestResource private final SearchExecutionGuard executionGuard; private final ExportJobService exportJobService; private final QueryValidationService queryValidationService; + private final Function, Stream> streamCategoryMapper; //allow mocking Function>, ChunkedOutput> asyncRunner = ChunkedRunner::runAsync; Function messagesExporterFactory; - Function, Stream> streamCategoryMapper; @Inject public MessagesResource( @@ -97,13 +97,24 @@ public MessagesResource( ExportJobService exportJobService, QueryValidationService queryValidationService, StreamService streamService) { + this(exporter, commandFactory, searchDomain, executionGuard, eventBus, exportJobService, queryValidationService, categories -> streamService.mapCategoriesToIds(categories).stream()); + } + + MessagesResource(MessagesExporter exporter, + CommandFactory commandFactory, + SearchDomain searchDomain, + SearchExecutionGuard executionGuard, + @SuppressWarnings("UnstableApiUsage") EventBus eventBus, + ExportJobService exportJobService, + QueryValidationService queryValidationService, + Function, Stream> streamCategoryMapper) { this.commandFactory = commandFactory; this.searchDomain = searchDomain; this.executionGuard = executionGuard; this.exportJobService = exportJobService; this.queryValidationService = queryValidationService; this.messagesExporterFactory = context -> new AuditingMessagesExporter(context, eventBus, exporter); - this.streamCategoryMapper = categories -> streamService.mapCategoriesToIds(categories).stream(); + this.streamCategoryMapper = streamCategoryMapper; } @ApiOperation( diff --git a/graylog2-server/src/test/java/org/graylog/plugins/views/search/rest/MessagesResourceTest.java b/graylog2-server/src/test/java/org/graylog/plugins/views/search/rest/MessagesResourceTest.java index 3e3709d9cae5..1c18ca59d143 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/views/search/rest/MessagesResourceTest.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/views/search/rest/MessagesResourceTest.java @@ -91,7 +91,7 @@ void setUp() { class MessagesTestResource extends MessagesResource { public MessagesTestResource(MessagesExporter exporter, CommandFactory commandFactory, SearchDomain searchDomain, SearchExecutionGuard executionGuard, PermittedStreams permittedStreams, ObjectMapper objectMapper, EventBus eventBus, QueryValidationService validationService) { - super(exporter, commandFactory, searchDomain, executionGuard, eventBus, mock(ExportJobService.class), validationService); + super(exporter, commandFactory, searchDomain, executionGuard, eventBus, mock(ExportJobService.class), validationService, categories -> Stream.of()); } @Nullable From a7d1e54c051b232cbf791d595f0135b7c563d5de Mon Sep 17 00:00:00 2001 From: Zack King Date: Tue, 13 Aug 2024 14:23:21 -0500 Subject: [PATCH 09/13] Revert unused changes --- .../graylog/plugins/views/search/export/CommandFactory.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/export/CommandFactory.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/export/CommandFactory.java index aa97ff3a69fc..0aa4eaabf826 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/export/CommandFactory.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/export/CommandFactory.java @@ -30,7 +30,6 @@ import java.util.Collections; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; public class CommandFactory { @@ -57,7 +56,6 @@ public ExportMessagesCommand buildFromRequest(MessagesRequest request) { public ExportMessagesCommand buildWithSearchOnly(Search search, ResultFormat resultFormat) { Query query = queryFrom(search); - final Set queryStreamIds = query.usedStreamIds(); return builderFrom(resultFormat) .timeRange(resultFormat.timerange().orElse(toAbsolute(query.timerange()))) @@ -67,7 +65,7 @@ public ExportMessagesCommand buildWithSearchOnly(Search search, ResultFormat res .flatMap(List::stream) .collect(Collectors.toList()) ) - .streams(queryStreamIds) + .streams(query.usedStreamIds()) .build(); } From 55efe86626badab5193cc7752856d23bdafacde5 Mon Sep 17 00:00:00 2001 From: Zack King Date: Thu, 15 Aug 2024 10:38:39 -0500 Subject: [PATCH 10/13] cl --- changelog/unreleased/pr-20110.toml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelog/unreleased/pr-20110.toml diff --git a/changelog/unreleased/pr-20110.toml b/changelog/unreleased/pr-20110.toml new file mode 100644 index 000000000000..00a6717495e3 --- /dev/null +++ b/changelog/unreleased/pr-20110.toml @@ -0,0 +1,4 @@ +type = "a" +message = "Added categories to Streams to allow Illuminate content to be scoped to multiple products." + +pulls = ["20110"] From 865a36d25e621b600f523813705044cf77704741 Mon Sep 17 00:00:00 2001 From: Zack King Date: Mon, 19 Aug 2024 08:45:28 -0500 Subject: [PATCH 11/13] Add enterprise issue to changelog --- changelog/unreleased/pr-20110.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog/unreleased/pr-20110.toml b/changelog/unreleased/pr-20110.toml index 00a6717495e3..f8c1a286322a 100644 --- a/changelog/unreleased/pr-20110.toml +++ b/changelog/unreleased/pr-20110.toml @@ -1,4 +1,5 @@ type = "a" message = "Added categories to Streams to allow Illuminate content to be scoped to multiple products." +issues = ["graylog-plugin-enterprise#7945"] pulls = ["20110"] From 0578852b35d6c85ec31e4a67d696203dd9f49c4e Mon Sep 17 00:00:00 2001 From: Zack King Date: Thu, 22 Aug 2024 12:13:25 -0500 Subject: [PATCH 12/13] Replace StreamCategoryFilters with StreamFilters in place instead of at top level --- .../graylog/plugins/views/search/Query.java | 56 ++++++- .../PluggableSearchNormalization.java | 2 +- .../search/filter/StreamCategoryFilter.java | 4 + .../plugins/views/search/QueryTest.java | 146 ++++++++++++++++++ 4 files changed, 201 insertions(+), 7 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java index 7bd1dac7126b..1f8f381068a9 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java @@ -54,6 +54,7 @@ import javax.annotation.Nullable; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -250,12 +251,55 @@ public Query addStreamsToFilter(Set streamIds) { return toBuilder().filter(newFilter).build(); } - public Query mapStreamCategoriesToIds(Function, Stream> categoryMappingFunction, - StreamPermissions streamPermissions) { - Set streamIds = categoryMappingFunction.apply(usedStreamCategories()) - .filter(streamPermissions::canReadStream) - .collect(toSet()); - return addStreamsToFilter(streamIds); + public Query replaceStreamCategoryFilters(Function, Stream> categoryMappingFunction, + StreamPermissions streamPermissions) { + if (filter() == null) { + return this; + } + return toBuilder() + .filter(streamCategoryToStreamFiltersRecursively(filter(), categoryMappingFunction, streamPermissions)) + .build(); + } + + private Filter streamCategoryToStreamFiltersRecursively(Filter filter, + Function, Stream> categoryMappingFunction, + StreamPermissions streamPermissions) { + if (filter.filters() == null || filter.filters().isEmpty()) { + return filter; + } + Set mappedFilters = new HashSet<>(); + for (Filter f : filter.filters()) { + Filter mappedFilter = streamCategoryToStreamFilter(f, categoryMappingFunction, streamPermissions); + if (mappedFilter != null) { + mappedFilter = streamCategoryToStreamFiltersRecursively(mappedFilter, categoryMappingFunction, streamPermissions); + mappedFilters.add(mappedFilter); + } + } + if (mappedFilters.isEmpty()) { + return null; + } + return filter.toGenericBuilder().filters(mappedFilters.stream().filter(Objects::nonNull).collect(toSet())).build(); + } + + private Filter streamCategoryToStreamFilter(Filter filter, + Function, Stream> categoryMappingFunction, + StreamPermissions streamPermissions) { + if (filter instanceof StreamCategoryFilter scf) { + String[] mappedStreamIds = categoryMappingFunction.apply(List.of(scf.category())) + .filter(streamPermissions::canReadStream) + .toArray(String[]::new); + if (mappedStreamIds.length == 0) { + return null; + } + // Replace the category with an OrFilter of stream IDs and then add any filters form the original filter + Filter streamFilter = StreamFilter.anyIdOf(mappedStreamIds).toGenericBuilder().build(); + if (filter.filters() != null) { + streamFilter = streamFilter.toGenericBuilder().filters(filter.filters()).build(); + } + return streamFilter; + } else { + return filter; + } } private Filter addStreamsTo(Filter filter, Set streamIds) { diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/normalization/PluggableSearchNormalization.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/normalization/PluggableSearchNormalization.java index ba92911e6ad9..d0d782acdd48 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/normalization/PluggableSearchNormalization.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/normalization/PluggableSearchNormalization.java @@ -103,7 +103,7 @@ public Query preValidation(final Query query, final ParameterProvider parameterP if (!query.hasStreams()) { normalizedQuery = query.addStreamsToFilter(searchUser.streams().loadMessageStreamsWithFallback()); } else if (!query.usedStreamCategories().isEmpty()) { - normalizedQuery = query.mapStreamCategoriesToIds(streamCategoryMapper, searchUser); + normalizedQuery = query.replaceStreamCategoryFilters(streamCategoryMapper, searchUser); } if (!executionState.equals(ExecutionState.empty())) { diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/filter/StreamCategoryFilter.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/filter/StreamCategoryFilter.java index feeec05a0122..d2fda3c1a7e5 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/filter/StreamCategoryFilter.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/filter/StreamCategoryFilter.java @@ -52,6 +52,10 @@ public static Builder builder() { public abstract Builder toBuilder(); + public static StreamCategoryFilter ofCategory(String category) { + return builder().category(category).build(); + } + @Override public Filter.Builder toGenericBuilder() { return toBuilder(); diff --git a/graylog2-server/src/test/java/org/graylog/plugins/views/search/QueryTest.java b/graylog2-server/src/test/java/org/graylog/plugins/views/search/QueryTest.java index e879b1347fa6..3a8e5d31275b 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/views/search/QueryTest.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/views/search/QueryTest.java @@ -34,6 +34,10 @@ import com.google.common.collect.ImmutableSet; import org.graylog.plugins.views.search.elasticsearch.ElasticsearchQueryString; import org.graylog.plugins.views.search.engine.BackendQuery; +import org.graylog.plugins.views.search.filter.AndFilter; +import org.graylog.plugins.views.search.filter.OrFilter; +import org.graylog.plugins.views.search.filter.QueryStringFilter; +import org.graylog.plugins.views.search.filter.StreamCategoryFilter; import org.graylog.plugins.views.search.filter.StreamFilter; import org.graylog.plugins.views.search.rest.ExecutionState; import org.graylog.plugins.views.search.rest.ExecutionStateGlobalOverride; @@ -56,12 +60,16 @@ import org.junit.jupiter.api.Test; import java.io.IOException; +import java.util.Collection; import java.util.Collections; +import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; @@ -344,4 +352,142 @@ void appliesProperQueryExecutionStateIfEmptyGlobalOverride() { assertThat(query.query().queryString()) .isEqualTo("query"); } + + @Test + void replaceStreamCategoryFiltersWithStreamFilters() { + var queryWithCategories = Query.builder() + .id("query1") + .query(ElasticsearchQueryString.of("*")) + .filter(AndFilter.and(StreamCategoryFilter.ofCategory("colors"), StreamCategoryFilter.ofCategory("numbers"))) + .build(); + + queryWithCategories = queryWithCategories.replaceStreamCategoryFilters(this::categoryMapping, streamId -> true); + Filter filter = queryWithCategories.filter(); + assertThat(filter).isInstanceOf(AndFilter.class); + assertThat(filter.filters()).isNotNull(); + // The two StreamCategoryFilters should have been replaced with two OrFilters of three StreamFilters + assertThat(filter.filters()).hasSize(2); + assertThat(filter.filters().stream()).allSatisfy(f -> { + assertThat(f).isInstanceOf(OrFilter.class); + assertThat(f.filters()).isNotEmpty(); + assertThat(f.filters()).hasSize(3); + assertThat(f.filters().stream()).allSatisfy(f2 -> { + assertThat(f2).isInstanceOf(StreamFilter.class); + assertThat(f2.filters()).isNull(); + }); + }); + } + + @Test + void replaceStreamCategoryFiltersLeavesOtherFiltersAlone() { + var queryWithCategories = Query.builder() + .id("query1") + .query(ElasticsearchQueryString.of("*")) + .filter(AndFilter.builder() + .filters(ImmutableSet.builder() + .add(OrFilter.or(StreamCategoryFilter.ofCategory("colors"), StreamCategoryFilter.ofCategory("numbers"))) + .add(QueryStringFilter.builder().query("source:localhost").build()) + .build()) + .build()) + .build(); + + queryWithCategories = queryWithCategories.replaceStreamCategoryFilters(this::categoryMapping, streamId -> true); + Filter filter = queryWithCategories.filter(); + assertThat(filter).isInstanceOf(AndFilter.class); + assertThat(filter.filters()).isNotNull(); + assertThat(filter.filters()).hasSize(2); + // The QueryStringFilter should have been left alone in the replacement + assertThat(filter.filters().stream()).satisfiesOnlyOnce(f -> { + assertThat(f).isInstanceOf(QueryStringFilter.class); + assertThat(f.filters()).isNull(); + assertThat(((QueryStringFilter) f).query()).isEqualTo("source:localhost"); + }); + // The OrFilter of StreamCategoryFilters should have been converted to an OrFilter of StreamFilters + assertThat(filter.filters().stream()).satisfiesOnlyOnce(f -> { + assertThat(f).isInstanceOf(OrFilter.class); + assertThat(f.filters()).isNotEmpty(); + assertThat(f.filters()).hasSize(2); + assertThat(f.filters().stream()).allSatisfy(f2 -> { + assertThat(f2).isInstanceOf(OrFilter.class); + assertThat(f2.filters()).isNotEmpty(); + assertThat(f2.filters()).hasSize(3); + assertThat(f2.filters().stream()).allSatisfy(f3 -> { + assertThat(f3).isInstanceOf(StreamFilter.class); + assertThat(f3.filters()).isNull(); + }); + }); + }); + } + + @Test + void replaceStreamCategoryFiltersRespectsPermissions() { + var queryWithCategories = Query.builder() + .id("query1") + .query(ElasticsearchQueryString.of("*")) + .filter(AndFilter.and(StreamCategoryFilter.ofCategory("colors"), StreamCategoryFilter.ofCategory("numbers"))) + .build(); + + queryWithCategories = queryWithCategories.replaceStreamCategoryFilters(this::categoryMapping, (streamId) -> List.of("blue", "red", "one", "two").contains(streamId)); + Filter filter = queryWithCategories.filter(); + assertThat(filter).isInstanceOf(AndFilter.class); + assertThat(filter.filters()).isNotNull(); + // The two StreamCategoryFilters should have been replaced with two OrFilters of two StreamFilters + assertThat(filter.filters()).hasSize(2); + assertThat(filter.filters().stream()).allSatisfy(f -> { + assertThat(f).isInstanceOf(OrFilter.class); + assertThat(f.filters()).isNotEmpty(); + assertThat(f.filters()).hasSize(2); + assertThat(f.filters().stream()).allSatisfy(f2 -> { + assertThat(f2).isInstanceOf(StreamFilter.class); + assertThat(f2.filters()).isNull(); + }); + }); + } + + @Test + void replacementLeavesNoFilters() { + var queryWithCategories = Query.builder() + .id("query1") + .query(ElasticsearchQueryString.of("*")) + .filter(AndFilter.and(StreamCategoryFilter.ofCategory("colors"), StreamCategoryFilter.ofCategory("numbers"))) + .build(); + + queryWithCategories = queryWithCategories.replaceStreamCategoryFilters(this::categoryMapping, (streamId) -> false); + Filter filter = queryWithCategories.filter(); + assertThat(filter).isNull(); + } + + @Test + void emptyReplacementFiltersAreRemoved() { + var queryWithCategories = Query.builder() + .id("query1") + .query(ElasticsearchQueryString.of("*")) + .filter(AndFilter.builder() + .filters(ImmutableSet.builder() + .add(OrFilter.or(StreamCategoryFilter.ofCategory("colors"), StreamCategoryFilter.ofCategory("numbers"))) + .add(QueryStringFilter.builder().query("source:localhost").build()) + .build()) + .build()) + .build(); + + queryWithCategories = queryWithCategories.replaceStreamCategoryFilters(this::categoryMapping, (streamId) -> false); + Filter filter = queryWithCategories.filter(); + assertThat(filter).isInstanceOf(AndFilter.class); + assertThat(filter.filters()).isNotNull(); + assertThat(filter.filters()).hasSize(1); + } + + private Stream categoryMapping(Collection categories) { + Set streams = new HashSet<>(); + if (categories.contains("colors")) { + streams.addAll(List.of("red", "yellow", "blue")); + } + if (categories.contains("numbers")) { + streams.addAll(List.of("one", "two", "three")); + } + if (categories.contains("animals")) { + streams.addAll(List.of("cat", "dog", "fox")); + } + return streams.stream(); + } } From 43a84e0c25dea56c14f103aacaaa46e2d23f5b9a Mon Sep 17 00:00:00 2001 From: Zack King Date: Fri, 23 Aug 2024 11:42:49 -0500 Subject: [PATCH 13/13] Move StreamCategory to StreamFilter logic and address other feedback --- .../graylog/plugins/views/search/Query.java | 26 +----- .../graylog/plugins/views/search/Search.java | 12 --- .../search/filter/StreamCategoryFilter.java | 22 +++++ .../plugins/views/search/QueryTest.java | 93 ++++++------------- .../filter/StreamCategoryFilterTest.java | 77 +++++++++++++++ 5 files changed, 130 insertions(+), 100 deletions(-) create mode 100644 graylog2-server/src/test/java/org/graylog/plugins/views/search/filter/StreamCategoryFilterTest.java diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java index 1f8f381068a9..7faeb0316e85 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/Query.java @@ -269,7 +269,10 @@ private Filter streamCategoryToStreamFiltersRecursively(Filter filter, } Set mappedFilters = new HashSet<>(); for (Filter f : filter.filters()) { - Filter mappedFilter = streamCategoryToStreamFilter(f, categoryMappingFunction, streamPermissions); + Filter mappedFilter = f; + if (f instanceof StreamCategoryFilter scf) { + mappedFilter = scf.toStreamFilter(categoryMappingFunction, streamPermissions); + } if (mappedFilter != null) { mappedFilter = streamCategoryToStreamFiltersRecursively(mappedFilter, categoryMappingFunction, streamPermissions); mappedFilters.add(mappedFilter); @@ -281,27 +284,6 @@ private Filter streamCategoryToStreamFiltersRecursively(Filter filter, return filter.toGenericBuilder().filters(mappedFilters.stream().filter(Objects::nonNull).collect(toSet())).build(); } - private Filter streamCategoryToStreamFilter(Filter filter, - Function, Stream> categoryMappingFunction, - StreamPermissions streamPermissions) { - if (filter instanceof StreamCategoryFilter scf) { - String[] mappedStreamIds = categoryMappingFunction.apply(List.of(scf.category())) - .filter(streamPermissions::canReadStream) - .toArray(String[]::new); - if (mappedStreamIds.length == 0) { - return null; - } - // Replace the category with an OrFilter of stream IDs and then add any filters form the original filter - Filter streamFilter = StreamFilter.anyIdOf(mappedStreamIds).toGenericBuilder().build(); - if (filter.filters() != null) { - streamFilter = streamFilter.toGenericBuilder().filters(filter.filters()).build(); - } - return streamFilter; - } else { - return filter; - } - } - private Filter addStreamsTo(Filter filter, Set streamIds) { final Filter streamIdFilter = StreamFilter.anyIdOf(streamIds.toArray(new String[]{})); if (filter == null) { diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/Search.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/Search.java index 76eb01a42d95..4eca94312a6a 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/Search.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/Search.java @@ -198,24 +198,12 @@ public Set usedStreamIds() { return Sets.union(queryStreamIds, searchTypeStreamIds); } - public Set usedStreamCategories() { - return queries().stream() - .map(Query::usedStreamCategories) - .reduce(Collections.emptySet(), Sets::union); - } - public Set streamIdsForPermissionsCheck() { return queries().stream() .map(Query::streamIdsForPermissionsCheck) .reduce(Collections.emptySet(), Sets::union); } - public Set streamCategoriesForPermissionsCheck() { - return queries().stream() - .map(Query::usedStreamCategories) - .reduce(Collections.emptySet(), Sets::union); - } - public Query queryForSearchType(String searchTypeId) { return queries().stream() .filter(q -> q.hasSearchType(searchTypeId)) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/filter/StreamCategoryFilter.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/filter/StreamCategoryFilter.java index d2fda3c1a7e5..e3e37732d1e3 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/filter/StreamCategoryFilter.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/filter/StreamCategoryFilter.java @@ -23,9 +23,14 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.google.auto.value.AutoValue; import org.graylog.plugins.views.search.Filter; +import org.graylog.plugins.views.search.permissions.StreamPermissions; import javax.annotation.Nullable; +import java.util.Collection; +import java.util.List; import java.util.Set; +import java.util.function.Function; +import java.util.stream.Stream; @AutoValue @JsonTypeName(StreamCategoryFilter.NAME) @@ -56,6 +61,23 @@ public static StreamCategoryFilter ofCategory(String category) { return builder().category(category).build(); } + public Filter toStreamFilter(Function, Stream> categoryMappingFunction, + StreamPermissions streamPermissions) { + String[] mappedStreamIds = categoryMappingFunction.apply(List.of(category())) + .filter(streamPermissions::canReadStream) + .toArray(String[]::new); + // If the streamPermissions do not allow for any of the streams to be read, nullify this filter. + if (mappedStreamIds.length == 0) { + return null; + } + // Replace this category with an OrFilter of stream IDs and then add filters if they exist. + Filter streamFilter = StreamFilter.anyIdOf(mappedStreamIds).toGenericBuilder().build(); + if (filters() != null) { + streamFilter = streamFilter.toGenericBuilder().filters(filters()).build(); + } + return streamFilter; + } + @Override public Filter.Builder toGenericBuilder() { return toBuilder(); diff --git a/graylog2-server/src/test/java/org/graylog/plugins/views/search/QueryTest.java b/graylog2-server/src/test/java/org/graylog/plugins/views/search/QueryTest.java index 3a8e5d31275b..6134caac2b1d 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/views/search/QueryTest.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/views/search/QueryTest.java @@ -60,19 +60,18 @@ import org.junit.jupiter.api.Test; import java.io.IOException; -import java.util.Collection; import java.util.Collections; -import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; +import java.util.function.Function; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class QueryTest { @@ -355,13 +354,17 @@ void appliesProperQueryExecutionStateIfEmptyGlobalOverride() { @Test void replaceStreamCategoryFiltersWithStreamFilters() { + StreamCategoryFilter colorCategory = mock(StreamCategoryFilter.class); + StreamCategoryFilter numberCategory = mock(StreamCategoryFilter.class); + when(colorCategory.toStreamFilter(any(), any())).thenReturn(StreamFilter.anyIdOf("red", "blue", "yellow")); + when(numberCategory.toStreamFilter(any(), any())).thenReturn(StreamFilter.anyIdOf("one", "two", "three")); var queryWithCategories = Query.builder() .id("query1") .query(ElasticsearchQueryString.of("*")) - .filter(AndFilter.and(StreamCategoryFilter.ofCategory("colors"), StreamCategoryFilter.ofCategory("numbers"))) + .filter(AndFilter.and(colorCategory, numberCategory)) .build(); - queryWithCategories = queryWithCategories.replaceStreamCategoryFilters(this::categoryMapping, streamId -> true); + queryWithCategories = queryWithCategories.replaceStreamCategoryFilters(mock(Function.class), streamId -> true); Filter filter = queryWithCategories.filter(); assertThat(filter).isInstanceOf(AndFilter.class); assertThat(filter.filters()).isNotNull(); @@ -380,18 +383,22 @@ void replaceStreamCategoryFiltersWithStreamFilters() { @Test void replaceStreamCategoryFiltersLeavesOtherFiltersAlone() { + StreamCategoryFilter colorCategory = mock(StreamCategoryFilter.class); + StreamCategoryFilter numberCategory = mock(StreamCategoryFilter.class); + when(colorCategory.toStreamFilter(any(), any())).thenReturn(StreamFilter.anyIdOf("red", "blue", "yellow")); + when(numberCategory.toStreamFilter(any(), any())).thenReturn(StreamFilter.anyIdOf("one", "two", "three")); var queryWithCategories = Query.builder() .id("query1") .query(ElasticsearchQueryString.of("*")) .filter(AndFilter.builder() .filters(ImmutableSet.builder() - .add(OrFilter.or(StreamCategoryFilter.ofCategory("colors"), StreamCategoryFilter.ofCategory("numbers"))) + .add(OrFilter.or(colorCategory, numberCategory)) .add(QueryStringFilter.builder().query("source:localhost").build()) .build()) .build()) .build(); - queryWithCategories = queryWithCategories.replaceStreamCategoryFilters(this::categoryMapping, streamId -> true); + queryWithCategories = queryWithCategories.replaceStreamCategoryFilters(mock(Function.class), streamId -> true); Filter filter = queryWithCategories.filter(); assertThat(filter).isInstanceOf(AndFilter.class); assertThat(filter.filters()).isNotNull(); @@ -402,92 +409,46 @@ void replaceStreamCategoryFiltersLeavesOtherFiltersAlone() { assertThat(f.filters()).isNull(); assertThat(((QueryStringFilter) f).query()).isEqualTo("source:localhost"); }); - // The OrFilter of StreamCategoryFilters should have been converted to an OrFilter of StreamFilters - assertThat(filter.filters().stream()).satisfiesOnlyOnce(f -> { - assertThat(f).isInstanceOf(OrFilter.class); - assertThat(f.filters()).isNotEmpty(); - assertThat(f.filters()).hasSize(2); - assertThat(f.filters().stream()).allSatisfy(f2 -> { - assertThat(f2).isInstanceOf(OrFilter.class); - assertThat(f2.filters()).isNotEmpty(); - assertThat(f2.filters()).hasSize(3); - assertThat(f2.filters().stream()).allSatisfy(f3 -> { - assertThat(f3).isInstanceOf(StreamFilter.class); - assertThat(f3.filters()).isNull(); - }); - }); - }); - } - - @Test - void replaceStreamCategoryFiltersRespectsPermissions() { - var queryWithCategories = Query.builder() - .id("query1") - .query(ElasticsearchQueryString.of("*")) - .filter(AndFilter.and(StreamCategoryFilter.ofCategory("colors"), StreamCategoryFilter.ofCategory("numbers"))) - .build(); - - queryWithCategories = queryWithCategories.replaceStreamCategoryFilters(this::categoryMapping, (streamId) -> List.of("blue", "red", "one", "two").contains(streamId)); - Filter filter = queryWithCategories.filter(); - assertThat(filter).isInstanceOf(AndFilter.class); - assertThat(filter.filters()).isNotNull(); - // The two StreamCategoryFilters should have been replaced with two OrFilters of two StreamFilters - assertThat(filter.filters()).hasSize(2); - assertThat(filter.filters().stream()).allSatisfy(f -> { - assertThat(f).isInstanceOf(OrFilter.class); - assertThat(f.filters()).isNotEmpty(); - assertThat(f.filters()).hasSize(2); - assertThat(f.filters().stream()).allSatisfy(f2 -> { - assertThat(f2).isInstanceOf(StreamFilter.class); - assertThat(f2.filters()).isNull(); - }); - }); } @Test void replacementLeavesNoFilters() { + StreamCategoryFilter colorCategory = mock(StreamCategoryFilter.class); + StreamCategoryFilter numberCategory = mock(StreamCategoryFilter.class); + when(colorCategory.toStreamFilter(any(), any())).thenReturn(null); + when(numberCategory.toStreamFilter(any(), any())).thenReturn(null); var queryWithCategories = Query.builder() .id("query1") .query(ElasticsearchQueryString.of("*")) - .filter(AndFilter.and(StreamCategoryFilter.ofCategory("colors"), StreamCategoryFilter.ofCategory("numbers"))) + .filter(AndFilter.and(colorCategory, numberCategory)) .build(); - queryWithCategories = queryWithCategories.replaceStreamCategoryFilters(this::categoryMapping, (streamId) -> false); + queryWithCategories = queryWithCategories.replaceStreamCategoryFilters(mock(Function.class), (streamId) -> false); Filter filter = queryWithCategories.filter(); assertThat(filter).isNull(); } @Test void emptyReplacementFiltersAreRemoved() { + StreamCategoryFilter colorCategory = mock(StreamCategoryFilter.class); + StreamCategoryFilter numberCategory = mock(StreamCategoryFilter.class); + when(colorCategory.toStreamFilter(any(), any())).thenReturn(null); + when(numberCategory.toStreamFilter(any(), any())).thenReturn(null); var queryWithCategories = Query.builder() .id("query1") .query(ElasticsearchQueryString.of("*")) .filter(AndFilter.builder() .filters(ImmutableSet.builder() - .add(OrFilter.or(StreamCategoryFilter.ofCategory("colors"), StreamCategoryFilter.ofCategory("numbers"))) + .add(OrFilter.or(colorCategory, numberCategory)) .add(QueryStringFilter.builder().query("source:localhost").build()) .build()) .build()) .build(); - queryWithCategories = queryWithCategories.replaceStreamCategoryFilters(this::categoryMapping, (streamId) -> false); + queryWithCategories = queryWithCategories.replaceStreamCategoryFilters(mock(Function.class), (streamId) -> false); Filter filter = queryWithCategories.filter(); assertThat(filter).isInstanceOf(AndFilter.class); assertThat(filter.filters()).isNotNull(); assertThat(filter.filters()).hasSize(1); } - - private Stream categoryMapping(Collection categories) { - Set streams = new HashSet<>(); - if (categories.contains("colors")) { - streams.addAll(List.of("red", "yellow", "blue")); - } - if (categories.contains("numbers")) { - streams.addAll(List.of("one", "two", "three")); - } - if (categories.contains("animals")) { - streams.addAll(List.of("cat", "dog", "fox")); - } - return streams.stream(); - } } diff --git a/graylog2-server/src/test/java/org/graylog/plugins/views/search/filter/StreamCategoryFilterTest.java b/graylog2-server/src/test/java/org/graylog/plugins/views/search/filter/StreamCategoryFilterTest.java new file mode 100644 index 000000000000..6069a0521155 --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog/plugins/views/search/filter/StreamCategoryFilterTest.java @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.plugins.views.search.filter; + +import org.graylog.plugins.views.search.Filter; +import org.junit.jupiter.api.Test; + +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +public class StreamCategoryFilterTest { + + @Test + void testToStreamFilter() { + Filter filter = StreamCategoryFilter.ofCategory("colors"); + filter = ((StreamCategoryFilter) filter).toStreamFilter(this::categoryMapping, streamId -> true); + + assertThat(filter).isInstanceOf(OrFilter.class); + assertThat(filter.filters()).isNotNull(); + assertThat(filter.filters()).hasSize(3); + assertThat(filter.filters().stream()).allSatisfy(f -> { + assertThat(f).isInstanceOf(StreamFilter.class); + assertThat(f.filters()).isNull(); + }); + } + + @Test + void testToStreamFilterWithPermissions() { + Filter filter = StreamCategoryFilter.ofCategory("colors"); + filter = ((StreamCategoryFilter) filter).toStreamFilter(this::categoryMapping, + (streamId) -> List.of("blue", "red", "one", "two").contains(streamId)); + + assertThat(filter).isInstanceOf(OrFilter.class); + assertThat(filter.filters()).isNotNull(); + assertThat(filter.filters()).hasSize(2); + assertThat(filter.filters().stream()).allSatisfy(f -> { + assertThat(f).isInstanceOf(StreamFilter.class); + assertThat(f.filters()).isNull(); + assertThat(List.of("blue", "red")).contains(((StreamFilter)f).streamId()); + }); + } + + @Test + void testToStreamFilterReturnsNull() { + Filter filter = StreamCategoryFilter.ofCategory("colors"); + filter = ((StreamCategoryFilter) filter).toStreamFilter(this::categoryMapping, (streamId) -> false); + + assertThat(filter).isNull(); + } + + private Stream categoryMapping(Collection categories) { + Set streams = new HashSet<>(); + if (categories.contains("colors")) { + streams.addAll(List.of("red", "yellow", "blue")); + } + return streams.stream(); + } +}