Enhanced Pipeline RuleUI (#15851)

* initial setup

* Rule Builder UI Help Icon (#15883)

* Extract function table into own component

* Migrate RuleHelper to typescript and functional component

* Add help icon and popover to rule block form

* generate simulator output

* Show function description in select list

* updated Cols to use xs

* Enhanced Simulator progress

* fix eslint issue

* add function groups and human readable names

backend for graylog/graylog-plugin-enterprise#5414

* Fix color for selected Option description and help box width

* set rulebuilder descriptor and primary param

* set rulebuilder descriptor and primary param

* set rulebuilder descriptor and primary param

* set rulebuilder descriptor and primary param

* fix: sort by rulebuilder name

* set rulebuilder descriptor and primary param

* remove boolean functions from actions

* Show output and use rule_builder_name

* repositioned Show Simulator

* display a simpler version of Rule Ref on RuleBuilder

* add SourceCodeEditor button

* fix reset simulation

* update css heights

* fix: generate simulator fields only on simulate

* put rule builder step variables into custom result node

* evaluate conditions in simulation

* Show select list for output variables for primary inputs

* Generate dynamic output variable list and show filtered list in fields

* Remove unused functions and code

* Don't add or update when required fields are missing and mark optional fields

* Fix proptype for RuleHelperTable

* Show variable names in Block Display

* Fix linter errors in Input component

* Fix input console error

* fix getDictForFunction undefined error

* Filter out variables without output and rename output var name

* readd handling of undefined dict in outputvariables

* Fix wrong license header in RuleBuilder

* integrated simulation api changes

* real time simulation

* show simulation conditions/actions outputs

* add condition groups

* fix failing tests, adjust to grouped conditions

* Rule Simulation label

* Remove params from display

* updated conditions/actions output keys

* add extractor fragments and fragment tests

* Show return type next to output

* Reduce spacing between output and block headline

* Fix existing tests and add new test cases

* Fix type errors and remove attrs usage in styled components

* add field type conditions, remove type check conditions, add metadata

* add date functions, add metadata

* added unit tests to simulator

* add string functions, function metadata

* unit test for simulator conditions/actions output

* fix missing name and conditions

* fixed tests

* direct navigation to rule builder after clicking create rule button

* add string conditions

* fix condition titles

* clean up conditions

* fix delete rule errors

* reposition sourceCodeEditor button and showSimulator switch

* Don't check for previousOutputPresent anymore

* Move Toggle to it's own component

* use theme spacing

* navigate to rules list onCancel instead of history.goBack

* add additional extractors and tests

* add grok matches condition

* Fix simulation result reset with the default simulation message when updating an action

* converted Rule.jsx to typescript

* Declare variableType as optional on type and propTypes

* fix grok matches and grok extractor, add tests

* update grok extractor title

* fix booleans in titles

* add suppress forbiddenapi usage in tests

* add split index extractor

* disable functions with incompatible return types and parameters

* disable functions with incompatible return types and parameters

* fix reenable set_field

* add lookup extractor

* add lookup extractor fragment

* added ConfirmNavigateToSourceCodeEditorModal

* Function typos, missing primary parameters

* Update logger name

Co-authored-by: Patrick Mann <[email protected]>

* completely remove deleted test

* fix logger name

* move `addFragment` upsert functionality to `RuleFragmentService`

* extract functions

---------

Co-authored-by: Laura Bergenthal-Grotlüschen <[email protected]>
Co-authored-by: Laura <[email protected]>
Co-authored-by: Matthias Oesterheld <[email protected]>
Co-authored-by: Matthias Oesterheld <[email protected]>
Co-authored-by: Patrick Mann <[email protected]>

graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionDescriptor.java

@@ -24,8 +24,10 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
+import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleBuilderFunctionGroup;
 
 import javax.annotation.Nullable;
+import java.util.Optional;
 
 @AutoValue
 @JsonAutoDetect
@@ -58,10 +60,50 @@ public ParameterDescriptor param(String name) {
     @JsonProperty
     public abstract boolean ruleBuilderEnabled();
 
+    @JsonIgnore
+    @Nullable
+    public abstract String ruleBuilderName();
+
+    /**
+     * default to function name
+     *
+     * @return function name, if rule builder is enabled and no name is explicitly set
+     */
+    @JsonProperty("rule_builder_name")
+    public String getRuleBuilderName() {
+        if (ruleBuilderEnabled() && ruleBuilderName() == null) {
+            return name();
+        }
+        return ruleBuilderName();
+    }
+
     @JsonProperty
     @Nullable
     public abstract String ruleBuilderTitle();
 
+    @JsonIgnore
+    @Nullable
+    public abstract RuleBuilderFunctionGroup ruleBuilderFunctionGroup();
+
+    /**
+     * tries to determine the function group from the primary parameter class, if not set
+     *
+     * @return determined group if rule builder is enabled and no group is explicitly set
+     */
+    @SuppressWarnings("rawtypes")
+    @JsonProperty("rule_builder_function_group")
+    public RuleBuilderFunctionGroup getRuleBuilderFunctionGroup() {
+        if (ruleBuilderEnabled() && ruleBuilderFunctionGroup() == null) {
+            final Optional<ParameterDescriptor> primaryParam = params().stream()
+                    .filter(ParameterDescriptor::primary)
+                    .findFirst();
+            if (primaryParam.isPresent()) {
+                return RuleBuilderFunctionGroup.map(primaryParam.get().type());
+            }
+        }
+        return ruleBuilderFunctionGroup();
+    }
+
     public static <T> Builder<T> builder() {
         //noinspection unchecked
         return new AutoValue_FunctionDescriptor.Builder().pure(false).ruleBuilderEnabled(false);
@@ -88,8 +130,12 @@ public Builder<T> ruleBuilderEnabled() {
             return ruleBuilderEnabled(true);
         }
 
+        public abstract Builder<T> ruleBuilderName(String ruleBuilderName);
+
         public abstract Builder<T> ruleBuilderTitle(String ruleBuilderTitle);
 
+        public abstract Builder<T> ruleBuilderFunctionGroup(RuleBuilderFunctionGroup ruleBuilderFunctionGroup);
+
         public Builder<T> params(ParameterDescriptor... params) {
             return params(ImmutableList.<ParameterDescriptor>builder().add(params).build());
         }
@@ -109,14 +155,18 @@ public static <T> FunctionDescriptor<T> createForRuleBuilder(
             @JsonProperty("return_type") Class<? extends T> returnType,
             @JsonProperty("params") @Nullable ImmutableList<ParameterDescriptor> params,
             @JsonProperty("description") @Nullable String description,
-            @JsonProperty("rule_builder_title") @Nullable String ruleBuilderTitle) {
+            @JsonProperty("rule_builder_name") @Nullable String ruleBuilderName,
+            @JsonProperty("rule_builder_title") @Nullable String ruleBuilderTitle,
+            @JsonProperty("rule_builder_group") @Nullable RuleBuilderFunctionGroup ruleBuilderFunctionGroup) {
         return FunctionDescriptor.<T>builder()
                 .name(name)
                 .returnType(returnType)
                 .params(params)
                 .description(description)
                 .ruleBuilderEnabled()
+                .ruleBuilderName(ruleBuilderName)
                 .ruleBuilderTitle(ruleBuilderTitle)
+                .ruleBuilderFunctionGroup(ruleBuilderFunctionGroup)
                 .build();
     }
 

graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java

@@ -21,6 +21,7 @@
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs;
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor;
 import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor;
+import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleBuilderFunctionGroup;
 import org.graylog2.plugin.IOState;
 import org.graylog2.plugin.inputs.MessageInput;
 import org.graylog2.shared.inputs.InputRegistry;
@@ -84,6 +85,10 @@ public FunctionDescriptor<Boolean> descriptor() {
                         idParam,
                         nameParam))
                 .description("Checks if a message arrived on a given input")
+                .ruleBuilderEnabled()
+                .ruleBuilderName("Check source input")
+                .ruleBuilderTitle("Check if message arrived on input<#if name??> '${name}'</#if><#if id??> '${id}'</#if>")
+                .ruleBuilderFunctionGroup(RuleBuilderFunctionGroup.BOOLEAN)
                 .build();
     }
 }

graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/GrokExists.java

@@ -22,6 +22,7 @@
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs;
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor;
 import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor;
+import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleBuilderFunctionGroup;
 import org.graylog2.grok.GrokPatternRegistry;
 
 import javax.inject.Inject;
@@ -73,7 +74,11 @@ public FunctionDescriptor<Boolean> descriptor() {
                .name(NAME)
                .returnType(Boolean.class)
                .params(of(patternParam, doLog))
-               .description("Checks if the given Grok pattern exists.")
+               .description("Checks if the given Grok pattern exists in Graylog.")
+               .ruleBuilderEnabled()
+               .ruleBuilderName("Check for grok pattern")
+               .ruleBuilderTitle("Check if grok pattern named '${pattern}' exists in Graylog")
+               .ruleBuilderFunctionGroup(RuleBuilderFunctionGroup.PATTERN)
                .build();
     }
 }

graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNotNull.java

@@ -21,6 +21,7 @@
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs;
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor;
 import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor;
+import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleBuilderFunctionGroup;
 
 import static com.google.common.collect.ImmutableList.of;
 
@@ -30,7 +31,7 @@ public class IsNotNull extends AbstractFunction<Boolean> {
     private final ParameterDescriptor<Object, Object> valueParam;
 
     public IsNotNull() {
-        valueParam = ParameterDescriptor.type("value", Object.class).description("The value to check").build();
+        valueParam = ParameterDescriptor.type("value", Object.class).primary().description("The value to check").build();
     }
 
     @Override
@@ -50,6 +51,10 @@ public FunctionDescriptor<Boolean> descriptor() {
                 .returnType(Boolean.class)
                 .params(of(valueParam))
                 .description("Checks whether a value is not 'null'")
+                .ruleBuilderEnabled(false)
+                .ruleBuilderName("Check if not null")
+                .ruleBuilderTitle("Check if value is not null")
+                .ruleBuilderFunctionGroup(RuleBuilderFunctionGroup.BOOLEAN)
                 .build();
     }
 }

graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNull.java

@@ -21,6 +21,7 @@
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs;
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor;
 import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor;
+import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleBuilderFunctionGroup;
 
 import static com.google.common.collect.ImmutableList.of;
 
@@ -30,7 +31,7 @@ public class IsNull extends AbstractFunction<Boolean> {
     private final ParameterDescriptor<Object, Object> valueParam;
 
     public IsNull() {
-        valueParam = ParameterDescriptor.type("value", Object.class).description("The value to check").build();
+        valueParam = ParameterDescriptor.type("value", Object.class).primary().description("The value to check").build();
     }
 
     @Override
@@ -50,6 +51,10 @@ public FunctionDescriptor<Boolean> descriptor() {
                 .returnType(Boolean.class)
                 .params(of(valueParam))
                 .description("Checks whether a value is 'null'")
+                .ruleBuilderEnabled(false)
+                .ruleBuilderName("Check if null")
+                .ruleBuilderTitle("Check if value is null")
+                .ruleBuilderFunctionGroup(RuleBuilderFunctionGroup.BOOLEAN)
                 .build();
     }
 }

graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/BooleanConversion.java

@@ -21,6 +21,7 @@
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs;
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor;
 import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor;
+import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleBuilderFunctionGroup;
 
 import static com.google.common.collect.ImmutableList.of;
 import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.bool;
@@ -34,7 +35,7 @@ public class BooleanConversion extends AbstractFunction<Boolean> {
 
 
     public BooleanConversion() {
-        valueParam = object("value").description("Value to convert").build();
+        valueParam = object("value").primary().description("Value to convert").build();
         defaultParam = bool("default").optional().description("Used when 'value' is null, defaults to false").build();
     }
 
@@ -54,6 +55,10 @@ public FunctionDescriptor<Boolean> descriptor() {
                 .returnType(Boolean.class)
                 .params(of(valueParam, defaultParam))
                 .description("Converts a value to a boolean value using its string representation")
+                .ruleBuilderEnabled(false)
+                .ruleBuilderName("Convert to boolean")
+                .ruleBuilderTitle("Convert '${value}' to boolean")
+                .ruleBuilderFunctionGroup(RuleBuilderFunctionGroup.CONVERSION)
                 .build();
     }
 }

graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/DoubleConversion.java

@@ -22,6 +22,7 @@
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs;
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor;
 import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor;
+import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleBuilderFunctionGroup;
 
 import static com.google.common.base.MoreObjects.firstNonNull;
 import static com.google.common.collect.ImmutableList.of;
@@ -38,7 +39,7 @@ public class DoubleConversion extends AbstractFunction<Double> {
     private final ParameterDescriptor<Double, Double> defaultParam;
 
     public DoubleConversion() {
-        valueParam = object(VALUE).description("Value to convert").build();
+        valueParam = object(VALUE).primary().description("Value to convert").build();
         defaultParam = floating(DEFAULT).optional().allowNegatives(true).description("Used when 'value' is null, defaults to 0").build();
     }
 
@@ -67,6 +68,10 @@ public FunctionDescriptor<Double> descriptor() {
                         defaultParam
                 ))
                 .description("Converts a value to a double value using its string representation")
+                .ruleBuilderEnabled(false)
+                .ruleBuilderName("Convert to double")
+                .ruleBuilderTitle("Convert '${value}' to double")
+                .ruleBuilderFunctionGroup(RuleBuilderFunctionGroup.CONVERSION)
                 .build();
     }
 }

graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsBoolean.java

@@ -21,6 +21,7 @@
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs;
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor;
 import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor;
+import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleBuilderFunctionGroup;
 
 import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object;
 
@@ -30,7 +31,7 @@ public class IsBoolean extends AbstractFunction<Boolean> {
     private final ParameterDescriptor<Object, Object> valueParam;
 
     public IsBoolean() {
-        valueParam = object("value").description("Value to check").build();
+        valueParam = object("value").primary().description("Value to check").build();
     }
 
     @Override
@@ -45,7 +46,11 @@ public FunctionDescriptor<Boolean> descriptor() {
                 .name(NAME)
                 .returnType(Boolean.class)
                 .params(valueParam)
-                .description("Checks whether a value is a boolean")
+                .description("Checks whether a value is a boolean value (true or false)")
+                .ruleBuilderEnabled(false)
+                .ruleBuilderName("Check if boolean")
+                .ruleBuilderTitle("Check if '${value}' is a boolean")
+                .ruleBuilderFunctionGroup(RuleBuilderFunctionGroup.BOOLEAN)
                 .build();
     }
 }

graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsCollection.java

@@ -21,6 +21,7 @@
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs;
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor;
 import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor;
+import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleBuilderFunctionGroup;
 
 import java.util.Collection;
 
@@ -32,7 +33,7 @@ public class IsCollection extends AbstractFunction<Boolean> {
     private final ParameterDescriptor<Object, Object> valueParam;
 
     public IsCollection() {
-        valueParam = object("value").description("Value to check").build();
+        valueParam = object("value").primary().description("Value to check").build();
     }
 
     @Override
@@ -47,7 +48,11 @@ public FunctionDescriptor<Boolean> descriptor() {
                 .name(NAME)
                 .returnType(Boolean.class)
                 .params(valueParam)
-                .description("Checks whether a value is a collection")
+                .description("Checks whether a value is an iterable collection")
+                .ruleBuilderEnabled(false)
+                .ruleBuilderName("Check if collection")
+                .ruleBuilderTitle("Check if '${value}' is a collection")
+                .ruleBuilderFunctionGroup(RuleBuilderFunctionGroup.BOOLEAN)
                 .build();
     }
 }

graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsDouble.java

@@ -21,6 +21,7 @@
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs;
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor;
 import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor;
+import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleBuilderFunctionGroup;
 
 import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object;
 
@@ -30,7 +31,7 @@ public class IsDouble extends AbstractFunction<Boolean> {
     private final ParameterDescriptor<Object, Object> valueParam;
 
     public IsDouble() {
-        valueParam = object("value").description("Value to check").build();
+        valueParam = object("value").primary().description("Value to check").build();
     }
 
     @Override
@@ -45,7 +46,11 @@ public FunctionDescriptor<Boolean> descriptor() {
                 .name(NAME)
                 .returnType(Boolean.class)
                 .params(valueParam)
-                .description("Checks whether a value is a double")
+                .description("Checks whether a value is a floating point value (of type double)")
+                .ruleBuilderEnabled(false)
+                .ruleBuilderName("Check if double")
+                .ruleBuilderTitle("Check if '${value}' is a double")
+                .ruleBuilderFunctionGroup(RuleBuilderFunctionGroup.BOOLEAN)
                 .build();
     }
 }

graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsList.java

@@ -21,6 +21,7 @@
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs;
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor;
 import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor;
+import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleBuilderFunctionGroup;
 
 import java.util.List;
 
@@ -32,7 +33,7 @@ public class IsList extends AbstractFunction<Boolean> {
     private final ParameterDescriptor<Object, Object> valueParam;
 
     public IsList() {
-        valueParam = object("value").description("Value to check").build();
+        valueParam = object("value").primary().description("Value to check").build();
     }
 
     @Override
@@ -48,6 +49,10 @@ public FunctionDescriptor<Boolean> descriptor() {
                 .returnType(Boolean.class)
                 .params(valueParam)
                 .description("Checks whether a value is a list")
+                .ruleBuilderEnabled(false)
+                .ruleBuilderName("Check if list")
+                .ruleBuilderTitle("Check if '${value}' is a list")
+                .ruleBuilderFunctionGroup(RuleBuilderFunctionGroup.BOOLEAN)
                 .build();
     }
 }

graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsLong.java

@@ -21,6 +21,7 @@
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs;
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor;
 import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor;
+import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleBuilderFunctionGroup;
 
 import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object;
 
@@ -30,7 +31,7 @@ public class IsLong extends AbstractFunction<Boolean> {
     private final ParameterDescriptor<Object, Object> valueParam;
 
     public IsLong() {
-        valueParam = object("value").description("Value to check").build();
+        valueParam = object("value").primary().description("Value to check").build();
     }
 
     @Override
@@ -46,6 +47,10 @@ public FunctionDescriptor<Boolean> descriptor() {
                 .returnType(Boolean.class)
                 .params(valueParam)
                 .description("Checks whether a value is a long integer")
+                .ruleBuilderEnabled(false)
+                .ruleBuilderName("Check if long")
+                .ruleBuilderTitle("Check if '${value}' is a long integer")
+                .ruleBuilderFunctionGroup(RuleBuilderFunctionGroup.BOOLEAN)
                 .build();
     }
 }