Display stream name and timestamp for search warm tier warning (#20427)

Co-authored-by: patrickmann <[email protected]> Co-authored-by: Mohamed OULD HOCINE <[email protected]>
Graylog2 · Sep 20, 2024 · df20f3d · df20f3d
1 parent 13d6f3e
commit df20f3d
Show file tree

Hide file tree

Showing 21 changed files with 164 additions and 24 deletions.
diff --git a/changelog/unreleased/pr-20427.toml b/changelog/unreleased/pr-20427.toml
@@ -0,0 +1,5 @@
+type = "c" # One of: a(dded), c(hanged), d(eprecated), r(emoved), f(ixed), s(ecurity)
+message = "Display Warm Tier Search warning as streams with timestamp."
+
+issues = ["graylog-plugin-enterprise#8402"]
+pulls = ["20427"]
diff --git a/...csearch7/src/main/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackend.java b/...csearch7/src/main/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackend.java
@@ -59,6 +59,7 @@
 import org.graylog2.indexer.ranges.IndexRange;
 import org.graylog2.plugin.Message;
 import org.graylog2.plugin.indexer.searches.timeranges.TimeRange;
+import org.graylog2.streams.StreamService;
 import org.joda.time.DateTimeZone;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -76,7 +77,6 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-import java.util.stream.Collectors;
 
 public class ElasticsearchBackend implements QueryBackend<ESGeneratedQueryContext> {
     private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchBackend.class);
@@ -88,6 +88,7 @@ public class ElasticsearchBackend implements QueryBackend<ESGeneratedQueryContex
     private final UsedSearchFiltersToQueryStringsMapper usedSearchFiltersToQueryStringsMapper;
     private final boolean allowLeadingWildcard;
     private final StatsCollector<QueryExecutionStats> executionStatsCollector;
+    private final StreamService streamService;
 
     @Inject
     public ElasticsearchBackend(Map<String, Provider<ESSearchTypeHandler<? extends SearchType>>> elasticsearchSearchTypeHandlers,
@@ -96,6 +97,7 @@ public ElasticsearchBackend(Map<String, Provider<ESSearchTypeHandler<? extends S
                                 ESGeneratedQueryContext.Factory queryContextFactory,
                                 UsedSearchFiltersToQueryStringsMapper usedSearchFiltersToQueryStringsMapper,
                                 StatsCollector<QueryExecutionStats> executionStatsCollector,
+                                StreamService streamService,
                                 @Named("allow_leading_wildcard_searches") boolean allowLeadingWildcard) {
         this.elasticsearchSearchTypeHandlers = elasticsearchSearchTypeHandlers;
         this.client = client;
@@ -104,6 +106,7 @@ public ElasticsearchBackend(Map<String, Provider<ESSearchTypeHandler<? extends S
         this.queryContextFactory = queryContextFactory;
         this.usedSearchFiltersToQueryStringsMapper = usedSearchFiltersToQueryStringsMapper;
         this.executionStatsCollector = executionStatsCollector;
+        this.streamService = streamService;
         this.allowLeadingWildcard = allowLeadingWildcard;
     }
 
@@ -226,6 +229,11 @@ public Set<IndexRange> indexRangesForStreamsInTimeRange(Set<String> streamIds, T
         return indexLookup.indexRangesForStreamsInTimeRange(streamIds, timeRange);
     }
 
+    @Override
+    public Optional<String> streamTitle(String streamId) {
+        return Optional.ofNullable(streamService.streamTitleFromCache(streamId));
+    }
+
     @WithSpan
     @Override
     public QueryResult doRun(SearchJob job, Query query, ESGeneratedQueryContext queryContext) {

diff --git a/.../java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackendErrorHandlingTest.java b/.../java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackendErrorHandlingTest.java
@@ -31,6 +31,7 @@
 import org.graylog.storage.elasticsearch7.testing.TestMultisearchResponse;
 import org.graylog.storage.elasticsearch7.views.searchtypes.ESSearchTypeHandler;
 import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange;
+import org.graylog2.streams.StreamService;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -73,6 +74,7 @@ public void setUp() throws Exception {
                 ViewsUtils.createTestContextFactory(),
                 usedSearchFilters -> Collections.emptySet(),
                 new NoOpStatsCollector<>(),
+                mock(StreamService.class),
                 false);
         when(indexLookup.indexNamesForStreamsInTimeRange(any(), any())).thenReturn(Collections.emptySet());
 

diff --git a/...rg/graylog/storage/elasticsearch7/views/ElasticsearchBackendGeneratedRequestTestBase.java b/...rg/graylog/storage/elasticsearch7/views/ElasticsearchBackendGeneratedRequestTestBase.java
@@ -42,6 +42,7 @@
 import org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange;
 import org.graylog2.plugin.indexer.searches.timeranges.InvalidRangeParametersException;
 import org.graylog2.plugin.indexer.searches.timeranges.TimeRange;
+import org.graylog2.streams.StreamService;
 import org.junit.Before;
 import org.junit.Rule;
 import org.mockito.ArgumentCaptor;
@@ -56,6 +57,7 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 
 public class ElasticsearchBackendGeneratedRequestTestBase extends ElasticsearchMockedClientTestBase {
@@ -90,6 +92,7 @@ public void setUpSUT() {
                         .map(inlineSf -> ((InlineQueryStringSearchFilter) inlineSf).queryString())
                         .collect(Collectors.toSet()),
                 new NoOpStatsCollector<>(),
+                mock(StreamService.class),
                 false);
     }
 

diff --git a/...rch7/src/test/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackendTest.java b/...rch7/src/test/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackendTest.java
@@ -54,6 +54,7 @@
 import org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange;
 import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange;
 import org.graylog2.plugin.indexer.searches.timeranges.TimeRange;
+import org.graylog2.streams.StreamService;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.junit.Before;
@@ -101,6 +102,7 @@ public void setup() {
                 ViewsUtils.createTestContextFactory(),
                 usedSearchFiltersToQueryStringsMapper,
                 new NoOpStatsCollector<>(),
+                mock(StreamService.class),
                 false);
     }
 

diff --git a/...org/graylog/storage/elasticsearch7/views/ElasticsearchBackendUsingCorrectIndicesTest.java b/...org/graylog/storage/elasticsearch7/views/ElasticsearchBackendUsingCorrectIndicesTest.java
@@ -38,6 +38,7 @@
 import org.graylog2.indexer.results.TestResultMessageFactory;
 import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange;
 import org.graylog2.plugin.indexer.searches.timeranges.TimeRange;
+import org.graylog2.streams.StreamService;
 import org.joda.time.DateTimeUtils;
 import org.joda.time.DateTimeZone;
 import org.junit.After;
@@ -57,6 +58,7 @@
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.graylog.storage.elasticsearch7.views.ViewsUtils.indicesOf;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
@@ -92,6 +94,7 @@ public void setupSUT() throws Exception {
                 ViewsUtils.createTestContextFactory(),
                 usedSearchFilters -> Collections.emptySet(),
                 new NoOpStatsCollector<>(),
+                mock(StreamService.class),
                 false);
     }
 

diff --git a/...ge-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/OpenSearchBackend.java b/...ge-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/OpenSearchBackend.java
@@ -61,6 +61,7 @@
 import org.graylog2.plugin.Message;
 import org.graylog2.plugin.indexer.searches.timeranges.TimeRange;
 import org.graylog2.plugin.streams.Stream;
+import org.graylog2.streams.StreamService;
 import org.joda.time.DateTimeZone;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -77,7 +78,6 @@
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
 public class OpenSearchBackend implements QueryBackend<OSGeneratedQueryContext> {
     private static final Logger LOG = LoggerFactory.getLogger(OpenSearchBackend.class);
@@ -89,6 +89,7 @@ public class OpenSearchBackend implements QueryBackend<OSGeneratedQueryContext>
     private final UsedSearchFiltersToQueryStringsMapper usedSearchFiltersToQueryStringsMapper;
     private final boolean allowLeadingWildcard;
     private final StatsCollector<QueryExecutionStats> executionStatsCollector;
+    private final StreamService streamService;
 
     @Inject
     public OpenSearchBackend(Map<String, Provider<OSSearchTypeHandler<? extends SearchType>>> elasticsearchSearchTypeHandlers,
@@ -97,6 +98,7 @@ public OpenSearchBackend(Map<String, Provider<OSSearchTypeHandler<? extends Sear
                              OSGeneratedQueryContext.Factory queryContextFactory,
                              UsedSearchFiltersToQueryStringsMapper usedSearchFiltersToQueryStringsMapper,
                              StatsCollector<QueryExecutionStats> executionStatsCollector,
+                             StreamService streamService,
                              @Named("allow_leading_wildcard_searches") boolean allowLeadingWildcard) {
         this.openSearchSearchTypeHandlers = elasticsearchSearchTypeHandlers;
         this.client = client;
@@ -105,6 +107,7 @@ public OpenSearchBackend(Map<String, Provider<OSSearchTypeHandler<? extends Sear
         this.queryContextFactory = queryContextFactory;
         this.usedSearchFiltersToQueryStringsMapper = usedSearchFiltersToQueryStringsMapper;
         this.executionStatsCollector = executionStatsCollector;
+        this.streamService = streamService;
         this.allowLeadingWildcard = allowLeadingWildcard;
     }
 
@@ -229,6 +232,11 @@ public Set<IndexRange> indexRangesForStreamsInTimeRange(Set<String> streamIds, T
         return indexLookup.indexRangesForStreamsInTimeRange(streamIds, timeRange);
     }
 
+    @Override
+    public Optional<String> streamTitle(String streamId) {
+        return Optional.ofNullable(streamService.streamTitleFromCache(streamId));
+    }
+
     @Override
     @WithSpan
     public QueryResult doRun(SearchJob job, Query query, OSGeneratedQueryContext queryContext) {

diff --git a/...c/test/java/org/graylog/storage/opensearch2/views/OpenSearchBackendErrorHandlingTest.java b/...c/test/java/org/graylog/storage/opensearch2/views/OpenSearchBackendErrorHandlingTest.java
@@ -31,6 +31,7 @@
 import org.graylog.storage.opensearch2.testing.TestMultisearchResponse;
 import org.graylog.storage.opensearch2.views.searchtypes.OSSearchTypeHandler;
 import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange;
+import org.graylog2.streams.StreamService;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -73,6 +74,7 @@ public void setUp() throws Exception {
                 ViewsUtils.createTestContextFactory(),
                 usedSearchFilters -> Collections.emptySet(),
                 new NoOpStatsCollector<>(),
+                mock(StreamService.class),
                 false);
         when(indexLookup.indexNamesForStreamsInTimeRange(any(), any())).thenReturn(Collections.emptySet());
 

diff --git a/...java/org/graylog/storage/opensearch2/views/OpenSearchBackendGeneratedRequestTestBase.java b/...java/org/graylog/storage/opensearch2/views/OpenSearchBackendGeneratedRequestTestBase.java
@@ -42,6 +42,7 @@
 import org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange;
 import org.graylog2.plugin.indexer.searches.timeranges.InvalidRangeParametersException;
 import org.graylog2.plugin.indexer.searches.timeranges.TimeRange;
+import org.graylog2.streams.StreamService;
 import org.junit.Before;
 import org.junit.Rule;
 import org.mockito.ArgumentCaptor;
@@ -56,6 +57,7 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 
 public class OpenSearchBackendGeneratedRequestTestBase extends OpensearchMockedClientTestBase {
@@ -90,6 +92,7 @@ public void setUpSUT() {
                         .map(inlineSf -> ((InlineQueryStringSearchFilter) inlineSf).queryString())
                         .collect(Collectors.toSet()),
                 new NoOpStatsCollector<>(),
+                mock(StreamService.class),
                 false);
     }
 

diff --git a/...pensearch2/src/test/java/org/graylog/storage/opensearch2/views/OpenSearchBackendTest.java b/...pensearch2/src/test/java/org/graylog/storage/opensearch2/views/OpenSearchBackendTest.java
@@ -54,6 +54,7 @@
 import org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange;
 import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange;
 import org.graylog2.plugin.indexer.searches.timeranges.TimeRange;
+import org.graylog2.streams.StreamService;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.junit.Before;
@@ -101,6 +102,7 @@ public void setup() {
                 ViewsUtils.createTestContextFactory(),
                 usedSearchFiltersToQueryStringsMapper,
                 new NoOpStatsCollector<>(),
+                mock(StreamService.class),
                 false);
     }
 

diff --git a/.../java/org/graylog/storage/opensearch2/views/OpenSearchBackendUsingCorrectIndicesTest.java b/.../java/org/graylog/storage/opensearch2/views/OpenSearchBackendUsingCorrectIndicesTest.java
@@ -38,6 +38,7 @@
 import org.graylog2.indexer.results.TestResultMessageFactory;
 import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange;
 import org.graylog2.plugin.indexer.searches.timeranges.TimeRange;
+import org.graylog2.streams.StreamService;
 import org.joda.time.DateTimeUtils;
 import org.joda.time.DateTimeZone;
 import org.junit.After;
@@ -57,6 +58,7 @@
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.graylog.storage.opensearch2.views.ViewsUtils.indicesOf;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
@@ -92,6 +94,7 @@ public void setupSUT() throws Exception {
                 ViewsUtils.createTestContextFactory(),
                 usedSearchFilters -> Collections.emptySet(),
                 new NoOpStatsCollector<>(),
+                mock(StreamService.class),
                 false);
     }
 

diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/ExplainResults.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/ExplainResults.java
@@ -20,6 +20,7 @@
 import org.graylog2.indexer.MongoIndexSet;
 import org.graylog2.indexer.ranges.IndexRange;
 
+import java.util.Collection;
 import java.util.Map;
 import java.util.Set;
 
@@ -34,13 +35,14 @@ public record QueryExplainResult(Map<String, ExplainResult> searchTypes) {
     public record ExplainResult(String queryString, Set<IndexRangeResult> searchedIndexRanges) {
     }
 
-    public record IndexRangeResult(String indexName, long begin, long end, boolean isWarmTiered) {
-        public IndexRangeResult(String indexName, long begin, long end) {
-            this(indexName, begin, end, MongoIndexSet.indexHasWarmInfix(indexName));
+    public record IndexRangeResult(String indexName, long begin, long end, boolean isWarmTiered,
+                                   Collection<String> streamNames) {
+        public IndexRangeResult(String indexName, long begin, long end, Collection<String> streamNames) {
+            this(indexName, begin, end, MongoIndexSet.indexHasWarmInfix(indexName), streamNames);
         }
 
-        public static IndexRangeResult fromIndexRange(IndexRange indexRange) {
-            return new IndexRangeResult(indexRange.indexName(), indexRange.begin().getMillis(), indexRange.end().getMillis());
+        public static IndexRangeResult fromIndexRange(IndexRange indexRange, Collection<String> streamNames) {
+            return new IndexRangeResult(indexRange.indexName(), indexRange.begin().getMillis(), indexRange.end().getMillis(), streamNames);
         }
     }
 }
diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/QueryBackend.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/QueryBackend.java
@@ -41,6 +41,8 @@
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
+import static org.graylog.plugins.views.search.ExplainResults.IndexRangeResult.fromIndexRange;
+
 /**
  * A search backend that is capable of generating and executing search jobs
  *
@@ -137,9 +139,17 @@ default ExplainResults.QueryExplainResult doExplain(SearchJob job, Query query,
         final ImmutableMap.Builder<String, ExplainResults.ExplainResult> builder = ImmutableMap.builder();
 
         query.searchTypes().forEach(s -> {
-            final Set<ExplainResults.IndexRangeResult> indicesForQuery = indexRangesForStreamsInTimeRange(
-                    query.effectiveStreams(s), query.effectiveTimeRange(s))
-                    .stream().map(ExplainResults.IndexRangeResult::fromIndexRange).collect(Collectors.toSet());
+            final Set<String> streamIds = query.effectiveStreams(s);
+            final Set<String> streamTitles = streamIds.stream()
+                    .map(this::streamTitle)
+                    .flatMap(Optional::stream)
+                    .filter(Objects::nonNull)
+                    .collect(Collectors.toSet());
+
+            final Set<ExplainResults.IndexRangeResult> indicesForQuery =
+                    indexRangesForStreamsInTimeRange(streamIds, query.effectiveTimeRange(s)).stream()
+                            .map(indexRange -> fromIndexRange(indexRange, streamTitles))
+                            .collect(Collectors.toSet());
             queryContext.getSearchTypeQueryString(s.id())
                     .ifPresent(queryString -> builder.put(s.id(), new ExplainResults.ExplainResult(queryString, indicesForQuery)));
         });
@@ -149,6 +159,8 @@ default ExplainResults.QueryExplainResult doExplain(SearchJob job, Query query,
 
     Set<IndexRange> indexRangesForStreamsInTimeRange(final Set<String> streamIds, final TimeRange timeRange);
 
+    Optional<String> streamTitle(String streamId);
+
     default boolean isSearchTypeWithError(T queryContext, String searchTypeId) {
         return queryContext.errors().stream()
                 .filter(q -> q instanceof SearchTypeError)

diff --git a/...2-server/src/main/java/org/graylog/plugins/views/search/rest/QueryValidationResource.java b/...2-server/src/main/java/org/graylog/plugins/views/search/rest/QueryValidationResource.java
@@ -46,18 +46,21 @@
 import org.graylog2.plugin.indexer.searches.timeranges.TimeRange;
 import org.graylog2.plugin.rest.PluginRestResource;
 import org.graylog2.shared.rest.resources.RestResource;
+import org.graylog2.streams.StreamService;
 
 import java.time.Instant;
 import java.util.Comparator;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static java.util.Comparator.naturalOrder;
 import static java.util.Comparator.nullsLast;
+import static org.graylog.plugins.views.search.ExplainResults.IndexRangeResult.fromIndexRange;
 import static org.graylog2.shared.rest.documentation.generator.Generator.CLOUD_VISIBLE;
 
 @RequiresAuthentication
@@ -67,16 +70,18 @@ public class QueryValidationResource extends RestResource implements PluginRestR
 
     private final QueryValidationService queryValidationService;
     private final Optional<StreamQueryExplainer> optionalStreamQueryExplainer;
-
     private final IndexLookup indexLookup;
+    private final StreamService streamService;
 
     @Inject
     public QueryValidationResource(final QueryValidationService queryValidationService,
                                    final Optional<StreamQueryExplainer> optionalStreamQueryExplainer,
-                                   final IndexLookup indexLookup) {
+                                   final IndexLookup indexLookup,
+                                   final StreamService streamService) {
         this.queryValidationService = queryValidationService;
         this.optionalStreamQueryExplainer = optionalStreamQueryExplainer;
         this.indexLookup = indexLookup;
+        this.streamService = streamService;
     }
 
     @POST
@@ -138,8 +143,13 @@ private ValidationRequest prepareRequest(ValidationRequestDTO validationRequest,
     }
 
     private Set<ExplainResults.IndexRangeResult> indexRanges(ValidationRequest request) {
+        final Set<String> streamTitles = request.streams().stream()
+                .map(streamService::streamTitleFromCache)
+                .filter(Objects::nonNull)
+                .collect(Collectors.toSet());
+
         return indexLookup.indexRangesForStreamsInTimeRange(request.streams(), request.timerange()).stream()
-                .map(ExplainResults.IndexRangeResult::fromIndexRange)
+                .map(indexRange -> fromIndexRange(indexRange, streamTitles))
                 .collect(Collectors.toSet());
     }