Reset cluster block after MessagesIT if memory usage blocks the cluster

Graylog2 · Jul 11, 2023 · 45c388d · 45c388d
1 parent 751eccf
commit 45c388d
Show file tree

Hide file tree

Showing 4 changed files with 31 additions and 0 deletions.
diff --git a/...ge-elasticsearch7/src/test/java/org/graylog/storage/elasticsearch7/testing/ClientES7.java b/...ge-elasticsearch7/src/test/java/org/graylog/storage/elasticsearch7/testing/ClientES7.java
@@ -24,6 +24,8 @@
 import com.github.rholder.retry.WaitStrategies;
 import com.google.common.collect.Streams;
 import org.graylog.shaded.elasticsearch7.org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
+import org.graylog.shaded.elasticsearch7.org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsRequest;
+import org.graylog.shaded.elasticsearch7.org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsResponse;
 import org.graylog.shaded.elasticsearch7.org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.graylog.shaded.elasticsearch7.org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.graylog.shaded.elasticsearch7.org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
@@ -248,6 +250,14 @@ private String[] existingIndices() {
                 .toArray(String[]::new);
     }
 
+
+    public String getSetting(String setting) {
+        final ClusterGetSettingsRequest req = new ClusterGetSettingsRequest();
+        final ClusterGetSettingsResponse response = client.execute((c, requestOptions) -> c.cluster().getSettings(req, requestOptions),
+                "Unable to read OS cluster setting: " + setting);
+        return response.getSetting(setting);
+    }
+
     @Override
     public void putSetting(String setting, String value) {
         final ClusterUpdateSettingsRequest request = new ClusterUpdateSettingsRequest();

diff --git a/...-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/testing/ClientOS2.java b/...-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/testing/ClientOS2.java
@@ -24,6 +24,8 @@
 import com.github.rholder.retry.WaitStrategies;
 import com.google.common.collect.Streams;
 import org.graylog.shaded.opensearch2.org.opensearch.action.admin.cluster.health.ClusterHealthRequest;
+import org.graylog.shaded.opensearch2.org.opensearch.action.admin.cluster.settings.ClusterGetSettingsRequest;
+import org.graylog.shaded.opensearch2.org.opensearch.action.admin.cluster.settings.ClusterGetSettingsResponse;
 import org.graylog.shaded.opensearch2.org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.graylog.shaded.opensearch2.org.opensearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.graylog.shaded.opensearch2.org.opensearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
@@ -262,6 +264,13 @@ public void putSetting(String setting, String value) {
                 "Unable to update OS cluster setting: " + setting + "=" + value);
     }
 
+    public String getSetting(String setting) {
+        final ClusterGetSettingsRequest req = new ClusterGetSettingsRequest();
+        final ClusterGetSettingsResponse response = client.execute((c, requestOptions) -> c.cluster().getSettings(req, requestOptions),
+                "Unable to read OS cluster setting: " + setting);
+        return response.getSetting(setting);
+    }
+
     public void putPersistentSetting(String setting, String value) {
         final ClusterUpdateSettingsRequest request = new ClusterUpdateSettingsRequest();
 

diff --git a/graylog2-server/src/test/java/org/graylog/testing/elasticsearch/Client.java b/graylog2-server/src/test/java/org/graylog/testing/elasticsearch/Client.java
@@ -61,6 +61,7 @@ default String createRandomIndex(String prefix) {
     String fieldType(String testIndexName, String source);
 
     void putSetting(String setting, String value);
+    String getSetting(String setting);
 
     void waitForIndexBlock(String index);
 

diff --git a/graylog2-server/src/test/java/org/graylog2/indexer/messages/MessagesIT.java b/graylog2-server/src/test/java/org/graylog2/indexer/messages/MessagesIT.java
@@ -23,6 +23,7 @@
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.graylog.failure.FailureSubmissionService;
 import org.graylog.testing.elasticsearch.ElasticsearchBaseTest;
+import org.graylog2.bootstrap.preflight.SearchDbPreflightCheck;
 import org.graylog2.indexer.IndexSet;
 import org.graylog2.indexer.results.ResultMessage;
 import org.graylog2.plugin.Message;
@@ -32,6 +33,8 @@
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.AbstractMap;
@@ -61,6 +64,8 @@ public abstract class MessagesIT extends ElasticsearchBaseTest {
 
     protected static final IndexSet indexSet = new MessagesTestIndexSet();
 
+    private static final Logger LOG = LoggerFactory.getLogger(MessagesIT.class);
+
     protected MessagesAdapter createMessagesAdapter() {
         return searchServer().adapters().messagesAdapter();
     }
@@ -79,6 +84,12 @@ public void setUp() throws Exception {
     @After
     public void tearDown() {
         client().cleanUp();
+        final String block = client().getSetting("cluster.blocks.create_index");
+        if(Boolean.parseBoolean(block)) {
+            // high memory usage in these tests may cause a cluster block. If that happens, we should reset this block after the test.
+            LOG.info("Indexer cluster is blocked after heavy memory usage, test done, resetting the block");
+            client().resetClusterBlock();
+        }
     }
 
     protected abstract boolean indexMessage(String index, Map<String, Object> source, @SuppressWarnings("SameParameterValue") String id);