Add CRC to sidecar default template (#17246)

* add CRC for default template

* add hardcode prior version CRCs

* update CL

* Only update if different

* log if ignoring an unknown, old template

* more legacy CRCs

* review feedback fixes

* handle PUT and POST correctly

* remove redundant CRC in request

changelog/unreleased/issue-17040.toml

@@ -0,0 +1,5 @@
+type = "c"
+message = "Enable sidecar default configurations to be updated on existing installs."
+
+issues = ["17040"]
+pulls = ["17246"]

graylog2-server/src/main/java/org/graylog/plugins/sidecar/migrations/V20180212165000_AddDefaultCollectors.java

@@ -35,6 +35,7 @@
 
 import javax.inject.Inject;
 import java.net.URI;
+import java.nio.charset.StandardCharsets;
 import java.time.ZonedDateTime;
 import java.util.Optional;
 import java.util.Set;
@@ -81,7 +82,6 @@ public ZonedDateTime createdAt() {
 
     @Override
     public void upgrade() {
-
         removeConfigPath();
         ensureConfigurationVariable("graylog_host", "Graylog Host.", httpExternalUri.getHost());
 
@@ -503,6 +503,23 @@ private Optional<Collector> ensureCollector(String collectorName,
                 LOG.debug(msg, collectorName, nodeOperatingSystem);
                 throw new IllegalArgumentException();
             }
+            if (!collector.defaultTemplateUpdated()) {
+                long newCRC = Collector.checksum(defaultTemplate.getBytes(StandardCharsets.UTF_8));
+                if (collector.defaultTemplateCRC() == null      // known obsolete version of template
+                        || newCRC != collector.defaultTemplateCRC() // new standard template
+                ) {
+                    LOG.info("{} collector default template on {} is unchanged, updating it.", collectorName, nodeOperatingSystem);
+                    try {
+                        return Optional.of(collectorService.save(
+                                collector.toBuilder()
+                                        .defaultTemplate(defaultTemplate)
+                                        .defaultTemplateCRC(newCRC)
+                                        .build()));
+                    } catch (Exception e) {
+                        LOG.error("Can't save collector '{}'!", collectorName, e);
+                    }
+                }
+            }
         } catch (IllegalArgumentException ignored) {
             LOG.info("{} collector on {} is missing, adding it.", collectorName, nodeOperatingSystem);
             try {
@@ -514,7 +531,8 @@ private Optional<Collector> ensureCollector(String collectorName,
                         executablePath,
                         executeParameters,
                         validationCommand,
-                        defaultTemplate
+                        defaultTemplate,
+                        Collector.checksum(defaultTemplate.getBytes(StandardCharsets.UTF_8))
                 )));
             } catch (Exception e) {
                 LOG.error("Can't save collector '{}'!", collectorName, e);
@@ -577,5 +595,4 @@ private void ensureDefaultConfiguration(String name, Collector collector) {
             LOG.error("Unable to access '{}' sidecar default configuration!", name);
         }
     }
-
 }

graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/models/Collector.java

@@ -16,16 +16,27 @@
  */
 package org.graylog.plugins.sidecar.rest.models;
 
-import com.fasterxml.jackson.annotation.*;
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.auto.value.AutoValue;
 import org.mongojack.Id;
 import org.mongojack.ObjectId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import javax.annotation.Nullable;
+import java.nio.charset.StandardCharsets;
+import java.util.Set;
+import java.util.zip.CRC32;
+import java.util.zip.Checksum;
 
 @AutoValue
 @JsonAutoDetect
 public abstract class Collector {
+    private static final Logger LOG = LoggerFactory.getLogger(Collector.class);
+
     public static final String FIELD_ID = "id";
     public static final String FIELD_NAME = "name";
     public static final String FIELD_SERVICE_TYPE = "service_type";
@@ -34,6 +45,30 @@ public abstract class Collector {
     public static final String FIELD_EXECUTE_PARAMETERS = "execute_parameters";
     public static final String FIELD_VALIDATION_PARAMETERS = "validation_parameters";
     public static final String FIELD_DEFAULT_TEMPLATE = "default_template";
+    public static final String FIELD_DEFAULT_TEMPLATE_CRC = "default_template_crc";
+
+    // Set of prior version CRCs for back-compat
+    private static final Set<Long> INITIAL_CRC = java.util.Set.of(
+            3280545580L, // 5.2 filebeat linux
+            3396210381L, // 5.2 filebeat darwin
+            3013497446L, // 5.2 filebeat freebsd
+            4009863009L, // 5.2 winlogbeat windows
+            2023247173L, // 5.2 nxlog linux
+            2491201449L, // 5.2 nxlog windows
+            2487909285L, // 5.2 auditbeat windows
+
+            4049210961L, // 5.1 and 5.0 filebeat linux/darwin/freebsd
+            2306685777L, // 5.1 and 5.0 winlogbeat windows
+            639836274L,  // 5.1 and 5.0 nxlog linux
+            2157898695L, // 5.1 and 5.0 nxlog windows
+            1490581247L, // 5.1 and 5.0 filebeat windows
+
+            1256873081L, // 4.3 filebeat linux
+            3852098581L, // 4.3 winlogbeat windows
+            3676599312L, // 4.3 nxlog linux
+            4293222217L, // 4.3 nxlog windows
+            2559816928L  // 4.3 filebeat windows
+    );
 
     @Id
     @ObjectId
@@ -66,6 +101,35 @@ public abstract class Collector {
     @Nullable
     public abstract String defaultTemplate();
 
+    @JsonProperty(FIELD_DEFAULT_TEMPLATE_CRC)
+    @Nullable
+    public abstract Long defaultTemplateCRC();
+
+    @JsonIgnore
+    public boolean defaultTemplateUpdated() {
+        if (defaultTemplate() == null) {
+            return false;
+        }
+
+        long crc = checksum(defaultTemplate().getBytes(StandardCharsets.UTF_8));
+        if (defaultTemplateCRC() == null) {
+            if (INITIAL_CRC.contains(crc)) {
+                return false; // known old version
+            } else {
+                LOG.info("{} collector default template on {} is an unrecognized version - not updating automatically.", name(), nodeOperatingSystem());
+                return true;  // changed or really old standard default template
+            }
+        }
+        return (crc != defaultTemplateCRC());
+    }
+
+    @JsonIgnore
+    public static long checksum(byte[] bytes) {
+        Checksum crc32 = new CRC32();
+        crc32.update(bytes, 0, bytes.length);
+        return crc32.getValue();
+    }
+
     public static Builder builder() {
         return new AutoValue_Collector.Builder();
     }
@@ -82,6 +146,7 @@ public abstract static class Builder {
         public abstract Builder executeParameters(String executeParameters);
         public abstract Builder validationParameters(String validationParameters);
         public abstract Builder defaultTemplate(String defaultTemplate);
+        public abstract Builder defaultTemplateCRC(Long checksum);
         public abstract Collector build();
     }
 
@@ -93,7 +158,8 @@ public static Collector create(@JsonProperty(FIELD_ID) @Nullable String id,
                                    @JsonProperty(FIELD_EXECUTABLE_PATH) String executablePath,
                                    @JsonProperty(FIELD_EXECUTE_PARAMETERS) @Nullable String executeParameters,
                                    @JsonProperty(FIELD_VALIDATION_PARAMETERS) @Nullable String validationParameters,
-                                   @JsonProperty(FIELD_DEFAULT_TEMPLATE) @Nullable String defaultTemplate) {
+                                   @JsonProperty(FIELD_DEFAULT_TEMPLATE) @Nullable String defaultTemplate,
+                                   @JsonProperty(FIELD_DEFAULT_TEMPLATE_CRC) @Nullable Long defaultTemplateCRC) {
         return builder()
                 .id(id)
                 .name(name)
@@ -103,6 +169,7 @@ public static Collector create(@JsonProperty(FIELD_ID) @Nullable String id,
                 .executeParameters(executeParameters)
                 .validationParameters(validationParameters)
                 .defaultTemplate(defaultTemplate)
+                .defaultTemplateCRC(defaultTemplateCRC)
                 .build();
     }
 }

graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/CollectorResource.java

@@ -204,13 +204,7 @@ public CollectorSummaryResponse listSummary(@ApiParam(name = "page") @QueryParam
     @AuditEvent(type = SidecarAuditEventTypes.COLLECTOR_CREATE)
     public Response createCollector(@ApiParam(name = "JSON body", required = true)
                                      @Valid @NotNull Collector request) throws BadRequestException {
-        Collector collector = collectorService.fromRequest(request);
-        final ValidationResult validationResult = validate(collector);
-        if (validationResult.failed()) {
-            return Response.status(Response.Status.BAD_REQUEST).entity(validationResult).build();
-        }
-        etagService.invalidateAllCollectors();
-        return Response.ok().entity(collectorService.save(collector)).build();
+        return saveCollector(collectorService.fromRequest(request));
     }
 
     @PUT
@@ -223,11 +217,22 @@ public Response updateCollector(@ApiParam(name = "id", required = true)
                                      @PathParam("id") String id,
                                      @ApiParam(name = "JSON body", required = true)
                                      @Valid @NotNull Collector request) throws BadRequestException {
-        Collector collector = collectorService.fromRequest(id, request);
+        return saveCollector(collectorService.fromRequest(id, request));
+    }
+
+    private Response saveCollector(Collector collector) {
         final ValidationResult validationResult = validate(collector);
         if (validationResult.failed()) {
             return Response.status(Response.Status.BAD_REQUEST).entity(validationResult).build();
         }
+
+        // Don't overwrite CRC of an existing collector. We need the original value to
+        // know that the entry has been modified.
+        Collector existingCollector = collectorService.findByNameAndOs(collector.name(), collector.nodeOperatingSystem());
+        if (existingCollector != null && existingCollector.defaultTemplateCRC() != null) {
+            collector = collector.toBuilder().defaultTemplateCRC(existingCollector.defaultTemplateCRC()).build();
+        }
+
         etagService.invalidateAllCollectors();
         return Response.ok().entity(collectorService.save(collector)).build();
     }

graylog2-server/src/main/java/org/graylog/plugins/sidecar/services/CollectorService.java

@@ -101,7 +101,8 @@ public Collector fromRequest(Collector request) {
                 request.executablePath(),
                 request.executeParameters(),
                 request.validationParameters(),
-                request.defaultTemplate());
+                request.defaultTemplate(),
+                null); // this is only ever written by 20180212165000_AddDefaultCollectors
     }
 
     public Collector fromRequest(String id, Collector request) {

graylog2-server/src/test/java/org/graylog/plugins/sidecar/collectors/SidecarServiceTest.java

@@ -365,7 +365,7 @@ private static Configuration getConfiguration() {
 
     private static Collector getCollector() {
         return Collector.create("collector-id", "collector-name", "service", "linux",
-                "/path", "param", "valid param", "");
+                "/path", "param", "valid param", "", null);
     }
 
     private Sidecar getTestSidecar() {