idaholab#302, allow user to include other suricata config YML files

Dockerfiles/suricata.Dockerfile

@@ -42,6 +42,7 @@ ENV YQ_URL "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_
 ENV SURICATA_CONFIG_DIR /etc/suricata
 ENV SURICATA_CONFIG_FILE "$SURICATA_CONFIG_DIR"/suricata.yaml
 ENV SURICATA_CUSTOM_RULES_DIR /opt/suricata/rules
+ENV SURICATA_CUSTOM_CONFIG_DIR /opt/suricata/include-configs
 ENV SURICATA_LOG_DIR /var/log/suricata
 ENV SURICATA_MANAGED_DIR /var/lib/suricata
 ENV SURICATA_MANAGED_RULES_DIR "$SURICATA_MANAGED_DIR/rules"
@@ -116,8 +117,8 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour
       usermod -a -G tty ${PUSER} && \
     ln -sfr /usr/local/bin/pcap_processor.py /usr/local/bin/pcap_suricata_processor.py && \
         (echo "*/5 * * * * /usr/local/bin/eve-clean-logs.sh\n0 */6 * * * /bin/bash /usr/local/bin/suricata-update-rules.sh\n" > ${SUPERCRONIC_CRONTAB}) && \
-    mkdir -p "$SURICATA_CUSTOM_RULES_DIR" && \
-        chown -R ${PUSER}:${PGROUP} "$SURICATA_CUSTOM_RULES_DIR" && \
+    mkdir -p "$SURICATA_CUSTOM_RULES_DIR" "$SURICATA_CUSTOM_CONFIG_DIR" && \
+        chown -R ${PUSER}:${PGROUP} "$SURICATA_CUSTOM_RULES_DIR" "$SURICATA_CUSTOM_CONFIG_DIR" && \
     cp "$(dpkg -L suricata-update | grep 'update\.yaml$' | head -n 1)" \
         "$SURICATA_UPDATE_CONFIG_FILE" && \
     find /tmp/default-rules/ -not -path '*/.gitignore' -type f -exec cp "{}" "$SURICATA_CONFIG_DIR"/rules/ \; && \
@@ -183,6 +184,7 @@ ENV PUSER_CHOWN "$SURICATA_CONFIG_DIR;$SURICATA_MANAGED_DIR;$SURICATA_LOG_DIR;$S
 
 VOLUME ["$SURICATA_CONFIG_DIR"]
 VOLUME ["$SURICATA_CUSTOM_RULES_DIR"]
+VOLUME ["$SURICATA_CUSTOM_CONFIG_DIR"]
 VOLUME ["$SURICATA_LOG_DIR"]
 VOLUME ["$SURICATA_MANAGED_DIR"]
 VOLUME ["$SURICATA_RUN_DIR"]

docker-compose-standalone.yml

@@ -189,7 +189,6 @@ services:
       start_period: 60s
   arkime:
     image: ghcr.io/idaholab/malcolm/arkime:23.11.0
-    # todo: viewer/wise in hedgehog profile (and what about nginx reaching back?)
     profiles: ["malcolm", "hedgehog"]
     logging: *default-logging
     restart: "no"
@@ -328,6 +327,7 @@ services:
       - ./suricata-logs:/var/log/suricata
       - ./pcap:/data/pcap
       - ./suricata/rules:/opt/suricata/rules:ro
+      - ./suricata/include-configs:/opt/suricata/include-configs:ro
     healthcheck:
       test: ["CMD", "supervisorctl", "status", "pcap-suricata"]
       interval: 30s
@@ -362,6 +362,7 @@ services:
       - ./nginx/ca-trust:/var/local/ca-trust:ro
       - ./suricata-logs:/var/log/suricata
       - ./suricata/rules:/opt/suricata/rules:ro
+      - ./suricata/include-configs:/opt/suricata/include-configs:ro
   file-monitor:
     image: ghcr.io/idaholab/malcolm/file-monitor:23.11.0
     profiles: ["malcolm", "hedgehog"]

docker-compose.yml

@@ -211,7 +211,6 @@ services:
       context: .
       dockerfile: Dockerfiles/arkime.Dockerfile
     image: ghcr.io/idaholab/malcolm/arkime:23.11.0
-    # todo: viewer/wise in hedgehog profile (and what about nginx reaching back?)
     profiles: ["malcolm", "hedgehog"]
     logging: *default-logging
     restart: "no"
@@ -364,6 +363,7 @@ services:
       - ./suricata-logs:/var/log/suricata
       - ./pcap:/data/pcap
       - ./suricata/rules:/opt/suricata/rules:ro
+      - ./suricata/include-configs:/opt/suricata/include-configs:ro
     healthcheck:
       test: ["CMD", "supervisorctl", "status", "pcap-suricata"]
       interval: 30s
@@ -401,6 +401,7 @@ services:
       - ./nginx/ca-trust:/var/local/ca-trust:ro
       - ./suricata-logs:/var/log/suricata
       - ./suricata/rules:/opt/suricata/rules:ro
+      - ./suricata/include-configs:/opt/suricata/include-configs:ro
   file-monitor:
     build:
       context: .

docs/contributing-local-modifications.md

@@ -60,10 +60,12 @@ suricata:
     - ./suricata-logs:/var/log/suricata
     - ./pcap:/data/pcap
     - ./suricata/rules:/opt/suricata/rules:ro
+    - ./suricata/include-configs:/opt/suricata/include-configs:ro
 suricata-live:
     - ./nginx/ca-trust:/var/local/ca-trust:ro
     - ./suricata-logs:/var/log/suricata
     - ./suricata/rules:/opt/suricata/rules:ro
+    - ./suricata/include-configs:/opt/suricata/include-configs:ro
 file-monitor:
     - ./nginx/ca-trust:/var/local/ca-trust:ro
     - ./zeek-logs/extract_files:/zeek/extract_files

kubernetes/11-suricata.yml

@@ -59,6 +59,8 @@ spec:
             name: suricata-offline-suricata-logs-volume
           - mountPath: "/opt/suricata/rules/configmap"
             name: suricata-offline-custom-rules-volume
+          - mountPath: "/opt/suricata/include-configs/configmap"
+            name: suricata-offline-custom-configs-volume
       initContainers:
       - name: suricata-offline-dirinit-container
         image: ghcr.io/idaholab/malcolm/dirinit:v23.11.0
@@ -86,4 +88,7 @@ spec:
             claimName: suricata-claim
         - name: suricata-offline-custom-rules-volume
           configMap:
-            name: suricata-rules
+            name: suricata-rules
+        - name: suricata-offline-custom-configs-volume
+          configMap:
+            name: suricata-configs

kubernetes/22-suricata-live.yml

@@ -49,6 +49,10 @@ spec:
             name: suricata-live-var-local-catrust-volume
           - mountPath: /var/log/suricata
             name: suricata-live-suricata-logs-volume
+          - mountPath: "/opt/suricata/rules/configmap"
+            name: suricata-live-custom-rules-volume
+          - mountPath: "/opt/suricata/include-configs/configmap"
+            name: suricata-live-custom-configs-volume
       initContainers:
       - name: suricata-live-dirinit-container
         image: ghcr.io/idaholab/malcolm/dirinit:v23.11.0
@@ -70,4 +74,10 @@ spec:
             name: var-local-catrust
         - name: suricata-live-suricata-logs-volume
           persistentVolumeClaim:
-            claimName: suricata-claim
+            claimName: suricata-claim
+        - name: suricata-live-custom-rules-volume
+          configMap:
+            name: suricata-rules
+        - name: suricata-live-custom-configs-volume
+          configMap:
+            name: suricata-configs

malcolm-iso/build.sh

@@ -114,6 +114,7 @@ if [ -d "$WORKDIR" ]; then
   mkdir -p "$MALCOLM_DEST_DIR/scripts/"
   mkdir -p "$MALCOLM_DEST_DIR/suricata-logs/live/"
   mkdir -p "$MALCOLM_DEST_DIR/suricata/rules/"
+  mkdir -p "$MALCOLM_DEST_DIR/suricata/include-configs/"
   mkdir -p "$MALCOLM_DEST_DIR/yara/rules/"
   mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/current/"
   mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/extract_files/preserved/"

scripts/malcolm_appliance_packager.sh

@@ -82,6 +82,7 @@ if mkdir "$DESTDIR"; then
   mkdir $VERBOSE -p "$DESTDIR/scripts/"
   mkdir $VERBOSE -p "$DESTDIR/suricata-logs/live/"
   mkdir $VERBOSE -p "$DESTDIR/suricata/rules/"
+  mkdir $VERBOSE -p "$DESTDIR/suricata/include-configs/"
   mkdir $VERBOSE -p "$DESTDIR/yara/rules/"
   mkdir $VERBOSE -p "$DESTDIR/zeek-logs/current/"
   mkdir $VERBOSE -p "$DESTDIR/zeek-logs/extract_files/preserved/"

scripts/malcolm_kubernetes.py

@@ -119,6 +119,12 @@
             'path': os.path.join(MalcolmPath, os.path.join('suricata', 'rules')),
         },
     ],
+    'suricata-configs': [
+        {
+            'secret': False,
+            'path': os.path.join(MalcolmPath, os.path.join('suricata', 'include-configs')),
+        },
+    ],
     'filebeat-certs': [
         {
             'secret': True,

scripts/malcolm_utils.py

@@ -545,6 +545,16 @@ def touch(filename):
     os.utime(filename, None)
 
 
+###################################################################################################
+# append strings to a text file
+def append_to_file(filename, value):
+    with open(filename, "a") as f:
+        if isinstance(value, Iterable) and not isinstance(value, str):
+            f.write('\n'.join(value))
+        else:
+            f.write(value)
+
+
 ###################################################################################################
 # read the contents of a file, first assuming text (with encoding), optionally falling back to binary
 def file_contents(filename, encoding='utf-8', binary_fallback=False):

sensor-iso/interface/sensor_ctl/supervisor.init/suricata_config_populate.sh

@@ -11,9 +11,11 @@ if [[ -n $SUPERVISOR_PATH ]] && [[ -r /usr/local/bin/suricata_config_populate.py
     [[ ! -f "$SUPERVISOR_PATH"/suricata/suricata.yaml ]] && cp /etc/suricata/suricata.yaml "$SUPERVISOR_PATH"/suricata/suricata.yaml
     [[ ! -f "$SUPERVISOR_PATH"/suricata/update.yaml ]] && cp "$(dpkg -L suricata-update | grep 'update\.yaml' | head -n 1)" "$SUPERVISOR_PATH"/suricata/update.yaml
 
-    # specify the custom rules directory relative to the supervisor path
+    # specify the custom rules and configuration directories relative to the supervisor path
     SURICATA_CUSTOM_RULES_DIR="$SUPERVISOR_PATH"/suricata/rules
+    SURICATA_CUSTOM_CONFIG_DIR="$SUPERVISOR_PATH"/suricata/include-configs
     [[ -d "$SURICATA_CUSTOM_RULES_DIR" ]] && export SURICATA_CUSTOM_RULES_DIR
+    [[ -d "$SURICATA_CUSTOM_CONFIG_DIR" ]] && export SURICATA_CUSTOM_CONFIG_DIR
 
     # all other arguments are controlled via environment variables sourced from control_vars.conf
     python3 /usr/local/bin/suricata_config_populate.py

sensor-iso/interface/sensor_ctl/supervisor.sh

@@ -81,7 +81,7 @@ mkdir -p "$SUPERVISOR_PATH/"{log,run}
 rm -f "$SUPERVISOR_PATH/"/log/*
 
 rm -rf /opt/sensor/sensor_ctl/zeek/intel/lock || true
-mkdir -p "$SUPERVISOR_PATH"/suricata/rules "$ZEEK_LOG_PATH"/suricata 2>/dev/null || true
+mkdir -p "$SUPERVISOR_PATH"/suricata/rules "$SUPERVISOR_PATH"/suricata/include-configs "$ZEEK_LOG_PATH"/suricata 2>/dev/null || true
 mkdir -p "$PCAP_PATH"/ 2>/dev/null || true
 mkdir -p "$SUPERVISOR_PATH"/supercronic 2>/dev/null && touch "$SUPERVISOR_PATH"/supercronic/crontab || true
 

shared/bin/sensor-init.sh

@@ -66,7 +66,7 @@ if [[ -r "$SCRIPT_PATH"/common-init.sh ]]; then
   if dpkg -s suricata >/dev/null 2>&1 ; then
     mkdir -p /etc/suricata/rules /var/log/suricata /var/lib/suricata/rules
     if [[ -d /opt/sensor/sensor_ctl ]]; then
-      mkdir -p /opt/sensor/sensor_ctl/suricata/rules
+      mkdir -p /opt/sensor/sensor_ctl/suricata/rules /opt/sensor/sensor_ctl/suricata/include-configs
       [[ ! -f /opt/sensor/sensor_ctl/suricata/suricata.yaml ]] && cp /etc/suricata/suricata.yaml /opt/sensor/sensor_ctl/suricata/suricata.yaml
       [[ ! -f /opt/sensor/sensor_ctl/suricata/update.yaml ]] && cp "$(dpkg -L suricata-update | grep 'update\.yaml' | head -n 1)" /opt/sensor/sensor_ctl/suricata/update.yaml
     fi

shared/bin/suricata_config_populate.py

@@ -27,7 +27,7 @@
 from shutil import move as MoveFile, copyfile as CopyFile
 from subprocess import PIPE, Popen
 
-from malcolm_utils import val2bool, deep_set, pushd, run_process
+from malcolm_utils import val2bool, deep_set, pushd, run_process, append_to_file
 
 ###################################################################################################
 args = None
@@ -49,26 +49,6 @@ def __call__(self, repr, data):
         return ret_val
 
 
-###################################################################################################
-def ObjToYamlStrLines(obj, options=None):
-    outputStr = None
-    if options is None:
-        options = {}
-
-    yaml = YAML()
-    yaml.preserve_quotes = False
-    yaml.representer.ignore_aliases = lambda x: True
-    yaml.representer.add_representer(type(None), NullRepresenter())
-    yaml.boolean_representation = ['no', 'yes']
-    yaml.version = YAML_VERSION
-
-    with StringIO() as stringStream:
-        yaml.dump(obj, stringStream, **options)
-        outputStr = stringStream.getvalue()
-
-    return outputStr.splitlines()
-
-
 ###################################################################################################
 
 DEFAULT_VARS = defaultdict(lambda: None)
@@ -542,6 +522,22 @@ def GetRuleSources(requireRulesExist=False):
     return ruleSources
 
 
+###################################################################################################
+def GetIncludeConfigSources():
+    global DEFAULT_VARS
+
+    configSources = list(
+        [
+            os.path.join(DEFAULT_VARS['CUSTOM_CONFIG_DIR'], x)
+            for x in fnmatch.filter(os.listdir(DEFAULT_VARS['CUSTOM_CONFIG_DIR']), '*.yaml')
+        ]
+        if DEFAULT_VARS['CUSTOM_CONFIG_DIR'] is not None
+        else []
+    )
+
+    return configSources
+
+
 ###################################################################################################
 def main():
     global args
@@ -643,6 +639,7 @@ def main():
         with open(args.input, 'r') as f:
             inYaml = YAML(typ='rt')
             inYaml.preserve_quotes = False
+            inYaml.allow_duplicate_keys = True
             inYaml.emitter.alt_null = None
             inYaml.representer.ignore_aliases = lambda x: True
             inYaml.boolean_representation = ['no', 'yes']
@@ -1073,20 +1070,41 @@ def main():
             os.path.join(DEFAULT_VARS['RUN_DIR'], 'suricata-command.socket'),
         )
 
+    extraConfigFiles = GetIncludeConfigSources()
+
     # validate suricata execution prior to calling it a day
     with tempfile.TemporaryDirectory() as tmpLogDir:
         with pushd(tmpLogDir):
             deep_set(cfg, ['stats', 'enabled'], True)
+
             cfg.pop('rule-files', None)
             deep_set(cfg, ['rule-files'], GetRuleSources(requireRulesExist=True))
+
+            # Hackety-hack, don't talk back! Despite the "Including multiple files" section of
+            #   https://docs.suricata.io/en/latest/configuration/includes.html#including-multiple-files
+            #   saying this can be set as an array, it does not seem to be working for me.
+            #   The sample suricata.yaml file shows it should be done like this:
+            #     include: include1.yaml
+            #     include: include2.yaml
+            # The reason this is a pain is that this is not actually valid YAML. So
+            #   what we are going to do is remove the 'include' section here,
+            #   write the YAML to a file, and then append the includes: afterwards
+            #   just in plain text.
+            cfg.pop('include', None)
+
             with open('suricata.yaml', 'w') as outTestFile:
                 outTestYaml = YAML(typ='rt')
                 outTestYaml.preserve_quotes = False
+                outTestYaml.allow_duplicate_keys = True
                 outTestYaml.representer.ignore_aliases = lambda x: True
                 outTestYaml.representer.add_representer(type(None), NullRepresenter())
                 outTestYaml.boolean_representation = ['no', 'yes']
                 outTestYaml.version = YAML_VERSION
                 outTestYaml.dump(cfg, outTestFile)
+
+            # see note on 'include' above
+            append_to_file('suricata.yaml', [''] + [f"include: {x}" for x in extraConfigFiles] + [''])
+
             script_return_code, output = run_process(
                 [
                     'suricata',
@@ -1109,18 +1127,25 @@ def main():
     cfg.pop('rule-files', None)
     deep_set(cfg, ['rule-files'], GetRuleSources(requireRulesExist=False))
 
+    # see note on 'include' above
+    cfg.pop('include', None)
+
     ##################################################################################################
 
     # write the new YAML file
     with open(args.output, 'w') as outfile:
         outYaml = YAML(typ='rt')
         outYaml.preserve_quotes = False
+        outYaml.allow_duplicate_keys = True
         outYaml.representer.ignore_aliases = lambda x: True
         outYaml.representer.add_representer(type(None), NullRepresenter())
         outYaml.boolean_representation = ['no', 'yes']
         outYaml.version = YAML_VERSION
         outYaml.dump(cfg, outfile)
 
+    # see note on 'include' above
+    append_to_file(args.output, [''] + [f"include: {x}" for x in extraConfigFiles] + [''])
+
     ##################################################################################################
 
     # remove the pidfile and command file for a new run (in case they weren't cleaned up before)

shared/bin/suricata_update_config_populate.py

@@ -49,26 +49,6 @@ def __call__(self, repr, data):
         return ret_val
 
 
-###################################################################################################
-def ObjToYamlStrLines(obj, options=None):
-    outputStr = None
-    if options is None:
-        options = {}
-
-    yaml = YAML()
-    yaml.preserve_quotes = False
-    yaml.representer.ignore_aliases = lambda x: True
-    yaml.representer.add_representer(type(None), NullRepresenter())
-    yaml.boolean_representation = ['no', 'yes']
-    yaml.version = YAML_VERSION
-
-    with StringIO() as stringStream:
-        yaml.dump(obj, stringStream, **options)
-        outputStr = stringStream.getvalue()
-
-    return outputStr.splitlines()
-
-
 ###################################################################################################
 
 DEFAULT_VARS = defaultdict(lambda: None)

suricata/include-configs/.gitignore

@@ -0,0 +1,3 @@
+*
+!.gitignore
+