diff --git a/changelog/unreleased/pr-20427.toml b/changelog/unreleased/pr-20427.toml new file mode 100644 index 000000000000..03c24d5f5128 --- /dev/null +++ b/changelog/unreleased/pr-20427.toml @@ -0,0 +1,5 @@ +type = "c" # One of: a(dded), c(hanged), d(eprecated), r(emoved), f(ixed), s(ecurity) +message = "Display Warm Tier Search warning as streams with timestamp." + +issues = ["graylog-plugin-enterprise#8402"] +pulls = ["20427"] diff --git a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackend.java b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackend.java index 9f549d13a74f..57000e47e390 100644 --- a/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackend.java +++ b/graylog-storage-elasticsearch7/src/main/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackend.java @@ -59,6 +59,7 @@ import org.graylog2.indexer.ranges.IndexRange; import org.graylog2.plugin.Message; import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; +import org.graylog2.streams.StreamService; import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,7 +77,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; public class ElasticsearchBackend implements QueryBackend { private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchBackend.class); @@ -88,6 +88,7 @@ public class ElasticsearchBackend implements QueryBackend executionStatsCollector; + private final StreamService streamService; @Inject public ElasticsearchBackend(Map>> elasticsearchSearchTypeHandlers, @@ -96,6 +97,7 @@ public ElasticsearchBackend(Map executionStatsCollector, + StreamService streamService, @Named("allow_leading_wildcard_searches") boolean allowLeadingWildcard) { this.elasticsearchSearchTypeHandlers = elasticsearchSearchTypeHandlers; this.client = client; @@ -104,6 +106,7 @@ public ElasticsearchBackend(Map indexRangesForStreamsInTimeRange(Set streamIds, T return indexLookup.indexRangesForStreamsInTimeRange(streamIds, timeRange); } + @Override + public Optional streamTitle(String streamId) { + return Optional.ofNullable(streamService.streamTitleFromCache(streamId)); + } + @WithSpan @Override public QueryResult doRun(SearchJob job, Query query, ESGeneratedQueryContext queryContext) { diff --git a/graylog-storage-elasticsearch7/src/test/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackendErrorHandlingTest.java b/graylog-storage-elasticsearch7/src/test/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackendErrorHandlingTest.java index bf556c674661..5ac3213bb5d6 100644 --- a/graylog-storage-elasticsearch7/src/test/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackendErrorHandlingTest.java +++ b/graylog-storage-elasticsearch7/src/test/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackendErrorHandlingTest.java @@ -31,6 +31,7 @@ import org.graylog.storage.elasticsearch7.testing.TestMultisearchResponse; import org.graylog.storage.elasticsearch7.views.searchtypes.ESSearchTypeHandler; import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange; +import org.graylog2.streams.StreamService; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -73,6 +74,7 @@ public void setUp() throws Exception { ViewsUtils.createTestContextFactory(), usedSearchFilters -> Collections.emptySet(), new NoOpStatsCollector<>(), + mock(StreamService.class), false); when(indexLookup.indexNamesForStreamsInTimeRange(any(), any())).thenReturn(Collections.emptySet()); diff --git a/graylog-storage-elasticsearch7/src/test/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackendGeneratedRequestTestBase.java b/graylog-storage-elasticsearch7/src/test/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackendGeneratedRequestTestBase.java index eb3956e776e8..09c377809019 100644 --- a/graylog-storage-elasticsearch7/src/test/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackendGeneratedRequestTestBase.java +++ b/graylog-storage-elasticsearch7/src/test/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackendGeneratedRequestTestBase.java @@ -42,6 +42,7 @@ import org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange; import org.graylog2.plugin.indexer.searches.timeranges.InvalidRangeParametersException; import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; +import org.graylog2.streams.StreamService; import org.junit.Before; import org.junit.Rule; import org.mockito.ArgumentCaptor; @@ -56,6 +57,7 @@ import java.util.Map; import java.util.stream.Collectors; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; public class ElasticsearchBackendGeneratedRequestTestBase extends ElasticsearchMockedClientTestBase { @@ -90,6 +92,7 @@ public void setUpSUT() { .map(inlineSf -> ((InlineQueryStringSearchFilter) inlineSf).queryString()) .collect(Collectors.toSet()), new NoOpStatsCollector<>(), + mock(StreamService.class), false); } diff --git a/graylog-storage-elasticsearch7/src/test/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackendTest.java b/graylog-storage-elasticsearch7/src/test/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackendTest.java index 6b829abe9d3b..1e916c1647ce 100644 --- a/graylog-storage-elasticsearch7/src/test/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackendTest.java +++ b/graylog-storage-elasticsearch7/src/test/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackendTest.java @@ -54,6 +54,7 @@ import org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange; import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange; import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; +import org.graylog2.streams.StreamService; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.junit.Before; @@ -101,6 +102,7 @@ public void setup() { ViewsUtils.createTestContextFactory(), usedSearchFiltersToQueryStringsMapper, new NoOpStatsCollector<>(), + mock(StreamService.class), false); } diff --git a/graylog-storage-elasticsearch7/src/test/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackendUsingCorrectIndicesTest.java b/graylog-storage-elasticsearch7/src/test/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackendUsingCorrectIndicesTest.java index e67b355463c0..2c48f4f932a8 100644 --- a/graylog-storage-elasticsearch7/src/test/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackendUsingCorrectIndicesTest.java +++ b/graylog-storage-elasticsearch7/src/test/java/org/graylog/storage/elasticsearch7/views/ElasticsearchBackendUsingCorrectIndicesTest.java @@ -38,6 +38,7 @@ import org.graylog2.indexer.results.TestResultMessageFactory; import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange; import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; +import org.graylog2.streams.StreamService; import org.joda.time.DateTimeUtils; import org.joda.time.DateTimeZone; import org.junit.After; @@ -57,6 +58,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.graylog.storage.elasticsearch7.views.ViewsUtils.indicesOf; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -92,6 +94,7 @@ public void setupSUT() throws Exception { ViewsUtils.createTestContextFactory(), usedSearchFilters -> Collections.emptySet(), new NoOpStatsCollector<>(), + mock(StreamService.class), false); } diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/OpenSearchBackend.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/OpenSearchBackend.java index 78fe70799a93..9d10cac412bc 100644 --- a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/OpenSearchBackend.java +++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/views/OpenSearchBackend.java @@ -61,6 +61,7 @@ import org.graylog2.plugin.Message; import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; import org.graylog2.plugin.streams.Stream; +import org.graylog2.streams.StreamService; import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,7 +78,6 @@ import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; public class OpenSearchBackend implements QueryBackend { private static final Logger LOG = LoggerFactory.getLogger(OpenSearchBackend.class); @@ -89,6 +89,7 @@ public class OpenSearchBackend implements QueryBackend private final UsedSearchFiltersToQueryStringsMapper usedSearchFiltersToQueryStringsMapper; private final boolean allowLeadingWildcard; private final StatsCollector executionStatsCollector; + private final StreamService streamService; @Inject public OpenSearchBackend(Map>> elasticsearchSearchTypeHandlers, @@ -97,6 +98,7 @@ public OpenSearchBackend(Map executionStatsCollector, + StreamService streamService, @Named("allow_leading_wildcard_searches") boolean allowLeadingWildcard) { this.openSearchSearchTypeHandlers = elasticsearchSearchTypeHandlers; this.client = client; @@ -105,6 +107,7 @@ public OpenSearchBackend(Map indexRangesForStreamsInTimeRange(Set streamIds, T return indexLookup.indexRangesForStreamsInTimeRange(streamIds, timeRange); } + @Override + public Optional streamTitle(String streamId) { + return Optional.ofNullable(streamService.streamTitleFromCache(streamId)); + } + @Override @WithSpan public QueryResult doRun(SearchJob job, Query query, OSGeneratedQueryContext queryContext) { diff --git a/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/views/OpenSearchBackendErrorHandlingTest.java b/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/views/OpenSearchBackendErrorHandlingTest.java index db9351d5aadc..93d53fddb2eb 100644 --- a/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/views/OpenSearchBackendErrorHandlingTest.java +++ b/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/views/OpenSearchBackendErrorHandlingTest.java @@ -31,6 +31,7 @@ import org.graylog.storage.opensearch2.testing.TestMultisearchResponse; import org.graylog.storage.opensearch2.views.searchtypes.OSSearchTypeHandler; import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange; +import org.graylog2.streams.StreamService; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -73,6 +74,7 @@ public void setUp() throws Exception { ViewsUtils.createTestContextFactory(), usedSearchFilters -> Collections.emptySet(), new NoOpStatsCollector<>(), + mock(StreamService.class), false); when(indexLookup.indexNamesForStreamsInTimeRange(any(), any())).thenReturn(Collections.emptySet()); diff --git a/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/views/OpenSearchBackendGeneratedRequestTestBase.java b/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/views/OpenSearchBackendGeneratedRequestTestBase.java index 6bbeb6578de3..d92da8673574 100644 --- a/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/views/OpenSearchBackendGeneratedRequestTestBase.java +++ b/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/views/OpenSearchBackendGeneratedRequestTestBase.java @@ -42,6 +42,7 @@ import org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange; import org.graylog2.plugin.indexer.searches.timeranges.InvalidRangeParametersException; import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; +import org.graylog2.streams.StreamService; import org.junit.Before; import org.junit.Rule; import org.mockito.ArgumentCaptor; @@ -56,6 +57,7 @@ import java.util.Map; import java.util.stream.Collectors; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; public class OpenSearchBackendGeneratedRequestTestBase extends OpensearchMockedClientTestBase { @@ -90,6 +92,7 @@ public void setUpSUT() { .map(inlineSf -> ((InlineQueryStringSearchFilter) inlineSf).queryString()) .collect(Collectors.toSet()), new NoOpStatsCollector<>(), + mock(StreamService.class), false); } diff --git a/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/views/OpenSearchBackendTest.java b/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/views/OpenSearchBackendTest.java index 806f0f828815..5748d6b0bc42 100644 --- a/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/views/OpenSearchBackendTest.java +++ b/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/views/OpenSearchBackendTest.java @@ -54,6 +54,7 @@ import org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange; import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange; import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; +import org.graylog2.streams.StreamService; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.junit.Before; @@ -101,6 +102,7 @@ public void setup() { ViewsUtils.createTestContextFactory(), usedSearchFiltersToQueryStringsMapper, new NoOpStatsCollector<>(), + mock(StreamService.class), false); } diff --git a/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/views/OpenSearchBackendUsingCorrectIndicesTest.java b/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/views/OpenSearchBackendUsingCorrectIndicesTest.java index 4247fbeec7a2..ae80965e06eb 100644 --- a/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/views/OpenSearchBackendUsingCorrectIndicesTest.java +++ b/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/views/OpenSearchBackendUsingCorrectIndicesTest.java @@ -38,6 +38,7 @@ import org.graylog2.indexer.results.TestResultMessageFactory; import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange; import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; +import org.graylog2.streams.StreamService; import org.joda.time.DateTimeUtils; import org.joda.time.DateTimeZone; import org.junit.After; @@ -57,6 +58,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.graylog.storage.opensearch2.views.ViewsUtils.indicesOf; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -92,6 +94,7 @@ public void setupSUT() throws Exception { ViewsUtils.createTestContextFactory(), usedSearchFilters -> Collections.emptySet(), new NoOpStatsCollector<>(), + mock(StreamService.class), false); } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/ExplainResults.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/ExplainResults.java index 108d6e1a95ce..218522396311 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/ExplainResults.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/ExplainResults.java @@ -20,6 +20,7 @@ import org.graylog2.indexer.MongoIndexSet; import org.graylog2.indexer.ranges.IndexRange; +import java.util.Collection; import java.util.Map; import java.util.Set; @@ -34,13 +35,14 @@ public record QueryExplainResult(Map searchTypes) { public record ExplainResult(String queryString, Set searchedIndexRanges) { } - public record IndexRangeResult(String indexName, long begin, long end, boolean isWarmTiered) { - public IndexRangeResult(String indexName, long begin, long end) { - this(indexName, begin, end, MongoIndexSet.indexHasWarmInfix(indexName)); + public record IndexRangeResult(String indexName, long begin, long end, boolean isWarmTiered, + Collection streamNames) { + public IndexRangeResult(String indexName, long begin, long end, Collection streamNames) { + this(indexName, begin, end, MongoIndexSet.indexHasWarmInfix(indexName), streamNames); } - public static IndexRangeResult fromIndexRange(IndexRange indexRange) { - return new IndexRangeResult(indexRange.indexName(), indexRange.begin().getMillis(), indexRange.end().getMillis()); + public static IndexRangeResult fromIndexRange(IndexRange indexRange, Collection streamNames) { + return new IndexRangeResult(indexRange.indexName(), indexRange.begin().getMillis(), indexRange.end().getMillis(), streamNames); } } } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/QueryBackend.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/QueryBackend.java index 5c83b3208f23..33bbbca9b18e 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/QueryBackend.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/engine/QueryBackend.java @@ -41,6 +41,8 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static org.graylog.plugins.views.search.ExplainResults.IndexRangeResult.fromIndexRange; + /** * A search backend that is capable of generating and executing search jobs * @@ -137,9 +139,17 @@ default ExplainResults.QueryExplainResult doExplain(SearchJob job, Query query, final ImmutableMap.Builder builder = ImmutableMap.builder(); query.searchTypes().forEach(s -> { - final Set indicesForQuery = indexRangesForStreamsInTimeRange( - query.effectiveStreams(s), query.effectiveTimeRange(s)) - .stream().map(ExplainResults.IndexRangeResult::fromIndexRange).collect(Collectors.toSet()); + final Set streamIds = query.effectiveStreams(s); + final Set streamTitles = streamIds.stream() + .map(this::streamTitle) + .flatMap(Optional::stream) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + + final Set indicesForQuery = + indexRangesForStreamsInTimeRange(streamIds, query.effectiveTimeRange(s)).stream() + .map(indexRange -> fromIndexRange(indexRange, streamTitles)) + .collect(Collectors.toSet()); queryContext.getSearchTypeQueryString(s.id()) .ifPresent(queryString -> builder.put(s.id(), new ExplainResults.ExplainResult(queryString, indicesForQuery))); }); @@ -149,6 +159,8 @@ default ExplainResults.QueryExplainResult doExplain(SearchJob job, Query query, Set indexRangesForStreamsInTimeRange(final Set streamIds, final TimeRange timeRange); + Optional streamTitle(String streamId); + default boolean isSearchTypeWithError(T queryContext, String searchTypeId) { return queryContext.errors().stream() .filter(q -> q instanceof SearchTypeError) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/rest/QueryValidationResource.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/rest/QueryValidationResource.java index 346ee2256523..ccad00f16a93 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/rest/QueryValidationResource.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/rest/QueryValidationResource.java @@ -46,11 +46,13 @@ import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; import org.graylog2.plugin.rest.PluginRestResource; import org.graylog2.shared.rest.resources.RestResource; +import org.graylog2.streams.StreamService; import java.time.Instant; import java.util.Comparator; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -58,6 +60,7 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet; import static java.util.Comparator.naturalOrder; import static java.util.Comparator.nullsLast; +import static org.graylog.plugins.views.search.ExplainResults.IndexRangeResult.fromIndexRange; import static org.graylog2.shared.rest.documentation.generator.Generator.CLOUD_VISIBLE; @RequiresAuthentication @@ -67,16 +70,18 @@ public class QueryValidationResource extends RestResource implements PluginRestR private final QueryValidationService queryValidationService; private final Optional optionalStreamQueryExplainer; - private final IndexLookup indexLookup; + private final StreamService streamService; @Inject public QueryValidationResource(final QueryValidationService queryValidationService, final Optional optionalStreamQueryExplainer, - final IndexLookup indexLookup) { + final IndexLookup indexLookup, + final StreamService streamService) { this.queryValidationService = queryValidationService; this.optionalStreamQueryExplainer = optionalStreamQueryExplainer; this.indexLookup = indexLookup; + this.streamService = streamService; } @POST @@ -138,8 +143,13 @@ private ValidationRequest prepareRequest(ValidationRequestDTO validationRequest, } private Set indexRanges(ValidationRequest request) { + final Set streamTitles = request.streams().stream() + .map(streamService::streamTitleFromCache) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + return indexLookup.indexRangesForStreamsInTimeRange(request.streams(), request.timerange()).stream() - .map(ExplainResults.IndexRangeResult::fromIndexRange) + .map(indexRange -> fromIndexRange(indexRange, streamTitles)) .collect(Collectors.toSet()); } diff --git a/graylog2-server/src/main/java/org/graylog2/streams/StreamService.java b/graylog2-server/src/main/java/org/graylog2/streams/StreamService.java index ff9cb13bbd0a..0ac34dd3b588 100644 --- a/graylog2-server/src/main/java/org/graylog2/streams/StreamService.java +++ b/graylog2-server/src/main/java/org/graylog2/streams/StreamService.java @@ -26,6 +26,7 @@ import org.graylog2.plugin.streams.StreamRule; import org.graylog2.rest.resources.streams.requests.CreateStreamRequest; +import javax.annotation.Nullable; import java.util.Collection; import java.util.List; import java.util.Map; @@ -60,6 +61,9 @@ default List loadAllByTitle(String title) { Map loadStreamTitles(Collection streamIds); + @Nullable + public String streamTitleFromCache(String streamId); + /** * @return the total number of streams */ diff --git a/graylog2-server/src/main/java/org/graylog2/streams/StreamServiceImpl.java b/graylog2-server/src/main/java/org/graylog2/streams/StreamServiceImpl.java index 986fa601bb74..c74ed318946f 100644 --- a/graylog2-server/src/main/java/org/graylog2/streams/StreamServiceImpl.java +++ b/graylog2-server/src/main/java/org/graylog2/streams/StreamServiceImpl.java @@ -17,6 +17,9 @@ package org.graylog2.streams; import com.google.common.base.Strings; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; @@ -26,6 +29,7 @@ import com.mongodb.DBObject; import com.mongodb.QueryBuilder; import com.mongodb.WriteResult; +import jakarta.annotation.Nonnull; import jakarta.inject.Inject; import org.bson.types.ObjectId; import org.graylog.security.entities.EntityOwnershipService; @@ -61,6 +65,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -70,6 +75,7 @@ import static com.mongodb.client.model.Projections.excludeId; import static com.mongodb.client.model.Projections.fields; import static com.mongodb.client.model.Projections.include; +import static org.graylog2.shared.utilities.StringUtils.f; import static org.graylog2.streams.StreamImpl.FIELD_ID; import static org.graylog2.streams.StreamImpl.FIELD_INDEX_SET_ID; import static org.graylog2.streams.StreamImpl.FIELD_TITLE; @@ -83,6 +89,8 @@ public class StreamServiceImpl extends PersistedServiceImpl implements StreamSer private final EntityOwnershipService entityOwnershipService; private final ClusterEventBus clusterEventBus; private final Set streamDeletionGuards; + private final CacheLoader streamTitleLoader; + private final LoadingCache streamTitleCache; @Inject public StreamServiceImpl(MongoConnection mongoConnection, @@ -101,6 +109,24 @@ public StreamServiceImpl(MongoConnection mongoConnection, this.entityOwnershipService = entityOwnershipService; this.clusterEventBus = clusterEventBus; this.streamDeletionGuards = streamDeletionGuards; + + this.streamTitleLoader = new CacheLoader() { + @Nonnull + @Override + public String load(@Nonnull String streamId) throws NotFoundException { + String title = loadStreamTitles(List.of(streamId)).get(streamId); + if (title != null) { + return title; + } else { + throw new NotFoundException(f("Couldn't find stream %s", streamId)); + } + } + }; + + this.streamTitleCache = CacheBuilder.newBuilder() + .expireAfterAccess(10, TimeUnit.SECONDS) + .build(streamTitleLoader); + } @Nullable @@ -196,6 +222,16 @@ public Map loadStreamTitles(Collection streamIds) { } } + @Override + @Nullable + public String streamTitleFromCache(String streamId) { + try { + return streamTitleCache.get(streamId); + } catch (Exception e) { + return null; + } + } + @Override public List loadAll() { return loadAll(Collections.emptyMap()); diff --git a/graylog2-server/src/test/java/org/graylog2/streams/StreamServiceImplTest.java b/graylog2-server/src/test/java/org/graylog2/streams/StreamServiceImplTest.java index 417fe5105597..e3fe1ae9cabc 100644 --- a/graylog2-server/src/test/java/org/graylog2/streams/StreamServiceImplTest.java +++ b/graylog2-server/src/test/java/org/graylog2/streams/StreamServiceImplTest.java @@ -96,6 +96,13 @@ public void loadStreamTitles() { .isInstanceOf(IllegalArgumentException.class); } + @Test + @MongoDBFixtures("someStreamsWithAlertConditions.json") + public void streamTitleFromCache() { + assertThat(streamService.streamTitleFromCache("565f02223b0c25a537197af2")).isEqualTo("Logins"); + assertThat(streamService.streamTitleFromCache("5628f4503b00deadbeef0002")).isNull(); + } + @Test @MongoDBFixtures("someStreamsWithoutAlertConditions.json") public void addOutputs() throws NotFoundException { diff --git a/graylog2-web-interface/src/views/components/contexts/SearchExplainContext.tsx b/graylog2-web-interface/src/views/components/contexts/SearchExplainContext.tsx index 32bfa17c1ba8..5b7f4b16a610 100644 --- a/graylog2-web-interface/src/views/components/contexts/SearchExplainContext.tsx +++ b/graylog2-web-interface/src/views/components/contexts/SearchExplainContext.tsx @@ -25,7 +25,8 @@ export type WidgetExplain = { index_name: string, begin: number, end: number, - is_warm_tiered: boolean + is_warm_tiered: boolean, + stream_names: Array }> } diff --git a/graylog2-web-interface/src/views/components/searchbar/queryvalidation/WarmTierErrorMessage.tsx b/graylog2-web-interface/src/views/components/searchbar/queryvalidation/WarmTierErrorMessage.tsx index 6139bdefd8af..b207311d825f 100644 --- a/graylog2-web-interface/src/views/components/searchbar/queryvalidation/WarmTierErrorMessage.tsx +++ b/graylog2-web-interface/src/views/components/searchbar/queryvalidation/WarmTierErrorMessage.tsx @@ -29,19 +29,37 @@ const WarmTierErrorMessage = ({ warmTierIndices } : Props) => { const formatTimestamp = (timestamp: number) : string => `${adjustFormat(new Date((timestamp)), 'default')}`; - const timestampInfo = warmTierIndices.map((warmTierIndex) => { - const begin = formatTimestamp(warmTierIndex.begin); - const end = formatTimestamp(warmTierIndex.end); + const streamsWithTimestamp = () : Array<{name: string, timestamp: number}> => { + const streamTimestampsList: {[key: string]: Array} = {}; - return `${begin} to ${end}`; - }); + warmTierIndices.forEach((index) => { + index.stream_names.forEach((streamName) => { + if (!streamTimestampsList[streamName]) { + streamTimestampsList[streamName] = [index.begin]; + } else { + streamTimestampsList[streamName].push(index.begin); + } + }); + }); - const timestampString = timestampInfo.join(', '); + return Object.entries(streamTimestampsList).map(([streamName, timestamps]) => { + const sortedTimestamps = timestamps.sort((a, b) => a - b); + const oldestTimestamp = sortedTimestamps[0]; + + return { name: streamName, timestamp: oldestTimestamp }; + }); + }; + + const streamsWithTimestampMap = streamsWithTimestamp(); + + if (streamsWithTimestampMap?.length <= 0) return null; return ( - The selected time range includes data stored in the Warm Tier, which can be slow to retrieve. - {timestampString.length > 0 && (` The following interval falls within the Warm Tier: ${timestampString}.`)} + The selected time range includes data stored in the Warm Tier, which can be slow to retrieve. Data older than the listed timestamp falls within the Warm Tier for that stream:
+ {streamsWithTimestampMap.map((streamWithTimestamp) => ( + <>{streamWithTimestamp.name}: {formatTimestamp(streamWithTimestamp.timestamp)}
+ ))} ); }; diff --git a/graylog2-web-interface/src/views/components/searchbar/queryvalidation/types.ts b/graylog2-web-interface/src/views/components/searchbar/queryvalidation/types.ts index e502dec2cd7a..f3b1a38446ab 100644 --- a/graylog2-web-interface/src/views/components/searchbar/queryvalidation/types.ts +++ b/graylog2-web-interface/src/views/components/searchbar/queryvalidation/types.ts @@ -19,8 +19,10 @@ export type IndexRange = { index_name: string, begin: number, end: number, - is_warm_tiered: boolean + is_warm_tiered: boolean, + stream_names: Array } + export type StreamDataRouting = { stream_name: string, stream_id: string, @@ -28,6 +30,7 @@ export type StreamDataRouting = { from: string, to: string, } + export type QueryValidationState = { status: 'OK' | 'ERROR' | 'WARNING' | 'INFO', explanations: Array<{ diff --git a/graylog2-web-interface/src/views/components/widgets/Widget.test.tsx b/graylog2-web-interface/src/views/components/widgets/Widget.test.tsx index 13495ff77e72..fa5af9a2d03f 100644 --- a/graylog2-web-interface/src/views/components/widgets/Widget.test.tsx +++ b/graylog2-web-interface/src/views/components/widgets/Widget.test.tsx @@ -49,12 +49,14 @@ const searchExplainContext = (searchedIndexRanges = [ begin: 1709716042283, end: 1709716342274, is_warm_tiered: false, + stream_names: ['foo', 'bar'], }, { index_name: 'aloho_1018', begin: 0, end: 0, is_warm_tiered: false, + stream_names: ['bar'], }, ], ) => ({ @@ -123,7 +125,8 @@ describe('', () => { index_name: string, begin: number, end: number, - is_warm_tiered: boolean + is_warm_tiered: boolean, + stream_names: Array }>, } @@ -382,18 +385,21 @@ describe('', () => { begin: 1709715731270, end: 1709716042255, is_warm_tiered: true, + stream_names: ['aloho', 'mora'], }, { index_name: 'aloho_1017', begin: 1709716042283, end: 1709716342274, is_warm_tiered: false, + stream_names: ['lumos'], }, { index_name: 'aloho_1018', begin: 0, end: 0, is_warm_tiered: false, + stream_names: [], }] } />);