Merge branch 'development' of https://github.com/mmguero-dev/Malcolm …

…into v23.12.0_merge_idaholab
idaholab · Dec 4, 2023 · 64cba5e · 64cba5e
2 parents 73f95a7 + eca0c86
commit 64cba5e
Show file tree

Hide file tree

Showing 6 changed files with 61 additions and 17 deletions.
diff --git a/docs/download.md b/docs/download.md
@@ -16,7 +16,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno
 
 | ISO | SHA256 |
 |---|---|
-| [malcolm-23.12.0.iso](/iso/malcolm-23.12.0.iso) (5.1GiB) |  [`da22b4bfab2ca8cb2a3ea6266b6cea603a9ae119de83009f3d133f0f202b566f`](/iso/malcolm-23.12.0.iso.sha256.txt) |
+| [malcolm-23.12.0.iso](/iso/malcolm-23.12.0.iso) (5.1GiB) |  [`3e836d09cd79a4e3f54c6fc365b032385312ad885b8483a0df156b59175d4909`](/iso/malcolm-23.12.0.iso.sha256.txt) |
 
 ## Hedgehog Linux
 

diff --git a/kubernetes/10-zeek.yml b/kubernetes/10-zeek.yml
@@ -63,12 +63,13 @@ spec:
           - mountPath: "/zeek/upload"
             name: zeek-offline-zeek-volume
             subPath: "upload"
+          - mountPath: "/opt/zeek/share/zeek/site/custom"
+            name: zeek-offline-custom-volume
+          - mountPath: "/opt/zeek/share/zeek/site/intel-preseed"
+            name: zeek-offline-intel-preseed-volume
           - mountPath: "/opt/zeek/share/zeek/site/intel"
-            name: zeek-offline-zeek-intel-and-config
+            name: zeek-offline-intel-volume
             subPath: "zeek/intel"
-          - mountPath: "/opt/zeek/share/zeek/site/custom"
-            name: zeek-offline-zeek-intel-and-config
-            subPath: "zeek/custom"
       initContainers:
       - name: zeek-offline-dirinit-container
         image: ghcr.io/idaholab/malcolm/dirinit:23.12.0
@@ -80,9 +81,9 @@ spec:
               name: process-env
         env:
           - name: PUSER_MKDIR
-            value: "/data/config:zeek/intel/MISP,zeek/intel/STIX,zeek/custom;/data/pcap:processed;/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload"
+            value: "/data/config:zeek/intel/MISP,zeek/intel/STIX;/data/pcap:processed;/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload"
         volumeMounts:
-          - name: zeek-offline-zeek-intel-and-config
+          - name: zeek-offline-intel-volume
             mountPath: "/data/config"
           - name: zeek-offline-pcap-volume
             mountPath: "/data/pcap"
@@ -98,6 +99,12 @@ spec:
         - name: zeek-offline-zeek-volume
           persistentVolumeClaim:
             claimName: zeek-claim
-        - name: zeek-offline-zeek-intel-and-config
+        - name: zeek-offline-custom-volume
+          configMap:
+            name: zeek-custom
+        - name: zeek-offline-intel-preseed-volume
+          configMap:
+            name: zeek-intel-preseed
+        - name: zeek-offline-intel-volume
           persistentVolumeClaim:
             claimName: config-claim
diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml
@@ -55,12 +55,13 @@ spec:
           - mountPath: "/zeek/upload"
             name: zeek-live-zeek-volume
             subPath: "upload"
+          - mountPath: "/opt/zeek/share/zeek/site/custom"
+            name: zeek-live-custom-volume
+          - mountPath: "/opt/zeek/share/zeek/site/intel-preseed"
+            name: zeek-live-intel-preseed-volume
           - mountPath: "/opt/zeek/share/zeek/site/intel"
-            name: zeek-live-zeek-intel-and-config
+            name: zeek-live-intel-volume
             subPath: "zeek/intel"
-          - mountPath: "/opt/zeek/share/zeek/site/custom"
-            name: zeek-live-zeek-intel-and-config
-            subPath: "zeek/custom"
       initContainers:
       - name: zeek-live-dirinit-container
         image: ghcr.io/idaholab/malcolm/dirinit:23.12.0
@@ -72,9 +73,9 @@ spec:
               name: process-env
         env:
           - name: PUSER_MKDIR
-            value: "/data/config:zeek/intel/MISP,zeek/intel/STIX,zeek/custom;/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload"
+            value: "/data/config:zeek/intel/MISP,zeek/intel/STIX;/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload"
         volumeMounts:
-          - name: zeek-live-zeek-intel-and-config
+          - name: zeek-live-intel-volume
             mountPath: "/data/config"
           - name: zeek-live-zeek-volume
             mountPath: "/data/zeek-logs"
@@ -85,6 +86,12 @@ spec:
         - name: zeek-live-zeek-volume
           persistentVolumeClaim:
             claimName: zeek-claim
-        - name: zeek-live-zeek-intel-and-config
+        - name: zeek-live-custom-volume
+          configMap:
+            name: zeek-custom
+        - name: zeek-live-intel-preseed-volume
+          configMap:
+            name: zeek-intel-preseed
+        - name: zeek-live-intel-volume
           persistentVolumeClaim:
             claimName: config-claim
diff --git a/scripts/control.py b/scripts/control.py
@@ -1015,6 +1015,7 @@ def start():
             ),
             BoundPath("zeek", "/zeek/extract_files", False, None, None),
             BoundPath("zeek", "/zeek/upload", False, None, None),
+            BoundPath("zeek", "/opt/zeek/share/zeek/site/custom", False, None, None),
             BoundPath("zeek", "/opt/zeek/share/zeek/site/intel", False, ["MISP", "STIX"], None),
             BoundPath("zeek-live", "/zeek/live", False, ["spool"], None),
             BoundPath("filebeat", "/zeek", False, ["processed", "current", "live", "extract_files", "upload"], None),

diff --git a/scripts/malcolm_kubernetes.py b/scripts/malcolm_kubernetes.py
@@ -159,6 +159,18 @@
             'path': os.path.join(MalcolmPath, os.path.join('htadmin', 'metadata')),
         },
     ],
+    'zeek-custom': [
+        {
+            'secret': False,
+            'path': os.path.join(MalcolmPath, os.path.join('zeek', 'custom')),
+        },
+    ],
+    'zeek-intel-preseed': [
+        {
+            'secret': False,
+            'path': os.path.join(MalcolmPath, os.path.join('zeek', 'intel')),
+        },
+    ],
 }
 
 # the PersistentVolumes themselves aren't used directly,

diff --git a/shared/bin/zeek_intel_setup.sh b/shared/bin/zeek_intel_setup.sh
@@ -17,6 +17,7 @@ ZEEK_INTEL_ITEM_EXPIRATION=${ZEEK_INTEL_ITEM_EXPIRATION:-"-1min"}
 ZEEK_INTEL_FEED_SINCE=${ZEEK_INTEL_FEED_SINCE:-""}
 ZEEK_INTEL_REFRESH_THREADS=${ZEEK_INTEL_REFRESH_THREADS:-"2"}
 INTEL_DIR=${INTEL_DIR:-"${ZEEK_DIR}/share/zeek/site/intel"}
+INTEL_PRESEED_DIR=${INTEL_PRESEED_DIR:-"${ZEEK_DIR}/share/zeek/site/intel-preseed"}
 THREAT_FEED_TO_ZEEK_SCRIPT=${THREAT_FEED_TO_ZEEK_SCRIPT:-"${ZEEK_DIR}/bin/zeek_intel_from_threat_feed.py"}
 LOCK_DIR="${INTEL_DIR}/lock"
 
@@ -29,6 +30,21 @@ mkdir -p -- "$(dirname "$LOCK_DIR")"
 if mkdir -- "$LOCK_DIR" 2>/dev/null; then
     trap finish EXIT
 
+    # if we have a directory to seed the intel config for the first time, start from a blank slate with just its contents
+    if [[ -d "${INTEL_DIR}" ]] && [[ -d "${INTEL_PRESEED_DIR}" ]]; then
+
+        EXCLUDES=()
+        EXCLUDES+=( --exclude='..*' )
+        EXCLUDES+=( --exclude='.dockerignore' )
+        EXCLUDES+=( --exclude='.gitignore' )
+        while read MAP_DIR; do
+            EXCLUDES+=( --exclude="${MAP_DIR}/" )
+        done < <(echo "${CONFIG_MAP_DIR:-configmap;secretmap}" | tr ';' '\n')
+
+        rsync --recursive --delete --delete-excluded "${EXCLUDES[@]}" "${INTEL_PRESEED_DIR}"/ "${INTEL_DIR}"/
+        mkdir -p "${INTEL_DIR}"/MISP "${INTEL_DIR}"/STIX || true
+    fi
+
     # create directive to @load every subdirectory in /opt/zeek/share/zeek/site/intel
     if [[ -d "${INTEL_DIR}" ]] && (( $(find "${INTEL_DIR}" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l) > 0 )); then
         pushd "${INTEL_DIR}" >/dev/null 2>&1
@@ -48,7 +64,8 @@ EOF
         THREAT_JSON_FILES=()
 
         # process subdirectories under INTEL_DIR
-        for DIR in $(find . -mindepth 1 -maxdepth 1 -type d 2>/dev/null); do
+        for DIR in $(find . -mindepth 1 -maxdepth 1 -type d 2>/dev/null | grep -v -P "$(echo "${CONFIG_MAP_DIR:-configmap;secretmap}" | sed 's/\(.*\)/^.\/(\1)$/' | tr ';' '|')"); do
+
             if [[ "${DIR}" == "./STIX" ]]; then
                 # this directory contains STIX JSON files we'll need to convert to zeek intel files then load
                 while IFS= read -r line; do
@@ -73,7 +90,7 @@ EOF
         done
 
         # process STIX and MISP inputs by converting them to Zeek intel format
-        if ( (( ${#THREAT_JSON_FILES[@]} )) || [[ -r ./STIX/.stix_input.txt ]] || [[ -r ./STIX/.misp_input.txt ]] ) && [[ -x "${THREAT_FEED_TO_ZEEK_SCRIPT}" ]]; then
+        if ( (( ${#THREAT_JSON_FILES[@]} )) || [[ -r ./STIX/.stix_input.txt ]] || [[ -r ./MISP/.misp_input.txt ]] ) && [[ -x "${THREAT_FEED_TO_ZEEK_SCRIPT}" ]]; then
             "${THREAT_FEED_TO_ZEEK_SCRIPT}" \
                 --since "${ZEEK_INTEL_FEED_SINCE}" \
                 --threads ${ZEEK_INTEL_REFRESH_THREADS} \