some code cleanup, moving things out of 'shared' that don't need to b…

…e in 'shared'

.github/workflows/api-build-and-push-ghcr.yml

@@ -9,16 +9,13 @@ on:
       - 'api/**'
       - 'Dockerfiles/api.Dockerfile'
       - 'shared/bin/*'
-      - '!shared/bin/agg-init.sh'
       - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
-      - '!shared/bin/sensor-init.sh'
       - '!shared/bin/os-disk-config.py'
       - '!shared/bin/extracted_files_http_server.py'
       - '!shared/bin/web-ui-asset-download.sh'
       - '!shared/bin/preseed_late_user_config.sh'
       - '!shared/bin/configure-interfaces.py'
-      - '!shared/bin/configure-capture.py'
       - '!shared/bin/suricata*'
       - '!shared/bin/zeek*'
       - '.trigger_workflow_build'

.github/workflows/arkime-build-and-push-ghcr.yml

@@ -9,16 +9,13 @@ on:
       - 'arkime/**'
       - 'Dockerfiles/arkime.Dockerfile'
       - 'shared/bin/*'
-      - '!shared/bin/agg-init.sh'
       - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
-      - '!shared/bin/sensor-init.sh'
       - '!shared/bin/extracted_files_http_server.py'
       - '!shared/bin/web-ui-asset-download.sh'
       - '!shared/bin/os-disk-config.py'
       - '!shared/bin/preseed_late_user_config.sh'
       - '!shared/bin/configure-interfaces.py'
-      - '!shared/bin/configure-capture.py'
       - '!shared/bin/suricata*'
       - '!shared/bin/zeek*'
       - '.trigger_workflow_build'

.github/workflows/dashboards-build-and-push-ghcr.yml

@@ -9,16 +9,13 @@ on:
       - 'dashboards/**'
       - 'Dockerfiles/dashboards.Dockerfile'
       - 'shared/bin/*'
-      - '!shared/bin/agg-init.sh'
       - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
-      - '!shared/bin/sensor-init.sh'
       - '!shared/bin/extracted_files_http_server.py'
       - '!shared/bin/web-ui-asset-download.sh'
       - '!shared/bin/os-disk-config.py'
       - '!shared/bin/preseed_late_user_config.sh'
       - '!shared/bin/configure-interfaces.py'
-      - '!shared/bin/configure-capture.py'
       - '!shared/bin/suricata*'
       - '!shared/bin/zeek*'
       - '.trigger_workflow_build'

.github/workflows/dashboards-helper-build-and-push-ghcr.yml

@@ -9,16 +9,13 @@ on:
       - 'dashboards/**'
       - 'Dockerfiles/dashboards-helper.Dockerfile'
       - 'shared/bin/*'
-      - '!shared/bin/agg-init.sh'
       - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
-      - '!shared/bin/sensor-init.sh'
       - '!shared/bin/extracted_files_http_server.py'
       - '!shared/bin/web-ui-asset-download.sh'
       - '!shared/bin/os-disk-config.py'
       - '!shared/bin/preseed_late_user_config.sh'
       - '!shared/bin/configure-interfaces.py'
-      - '!shared/bin/configure-capture.py'
       - '!shared/bin/suricata*'
       - '!shared/bin/zeek*'
       - '.trigger_workflow_build'

.github/workflows/file-monitor-build-and-push-ghcr.yml

@@ -9,13 +9,10 @@ on:
       - 'file-monitor/**'
       - 'Dockerfiles/file-monitor.Dockerfile'
       - 'shared/bin/*'
-      - '!shared/bin/agg-init.sh'
       - '!shared/bin/common-init.sh'
-      - '!shared/bin/sensor-init.sh'
       - '!shared/bin/os-disk-config.py'
       - '!shared/bin/preseed_late_user_config.sh'
       - '!shared/bin/configure-interfaces.py'
-      - '!shared/bin/configure-capture.py'
       - '!shared/bin/suricata*'
       - '!shared/bin/zeek*.sh'
       - '.trigger_workflow_build'

.github/workflows/file-upload-build-and-push-ghcr.yml

@@ -9,16 +9,13 @@ on:
       - 'file-upload/**'
       - 'Dockerfiles/file-upload.Dockerfile'
       - 'shared/bin/*'
-      - '!shared/bin/agg-init.sh'
       - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
-      - '!shared/bin/sensor-init.sh'
       - '!shared/bin/extracted_files_http_server.py'
       - '!shared/bin/web-ui-asset-download.sh'
       - '!shared/bin/os-disk-config.py'
       - '!shared/bin/preseed_late_user_config.sh'
       - '!shared/bin/configure-interfaces.py'
-      - '!shared/bin/configure-capture.py'
       - '!shared/bin/suricata*'
       - '!shared/bin/zeek*'
       - '.trigger_workflow_build'

.github/workflows/filebeat-build-and-push-ghcr.yml

@@ -9,16 +9,13 @@ on:
       - 'filebeat/**'
       - 'Dockerfiles/filebeat.Dockerfile'
       - 'shared/bin/*'
-      - '!shared/bin/agg-init.sh'
       - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
-      - '!shared/bin/sensor-init.sh'
       - '!shared/bin/extracted_files_http_server.py'
       - '!shared/bin/web-ui-asset-download.sh'
       - '!shared/bin/os-disk-config.py'
       - '!shared/bin/preseed_late_user_config.sh'
       - '!shared/bin/configure-interfaces.py'
-      - '!shared/bin/configure-capture.py'
       - '!shared/bin/suricata*'
       - '!shared/bin/zeek*'
       - '.trigger_workflow_build'

.github/workflows/freq-build-and-push-ghcr.yml

@@ -9,16 +9,13 @@ on:
       - 'freq-server/**'
       - 'Dockerfiles/freq.Dockerfile'
       - 'shared/bin/*'
-      - '!shared/bin/agg-init.sh'
       - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
-      - '!shared/bin/sensor-init.sh'
       - '!shared/bin/extracted_files_http_server.py'
       - '!shared/bin/web-ui-asset-download.sh'
       - '!shared/bin/os-disk-config.py'
       - '!shared/bin/preseed_late_user_config.sh'
       - '!shared/bin/configure-interfaces.py'
-      - '!shared/bin/configure-capture.py'
       - '!shared/bin/suricata*'
       - '!shared/bin/zeek*'
       - '.trigger_workflow_build'

.github/workflows/htadmin-build-and-push-ghcr.yml

@@ -9,16 +9,13 @@ on:
       - 'htadmin/**'
       - 'Dockerfiles/htadmin.Dockerfile'
       - 'shared/bin/*'
-      - '!shared/bin/agg-init.sh'
       - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
-      - '!shared/bin/sensor-init.sh'
       - '!shared/bin/extracted_files_http_server.py'
       - '!shared/bin/web-ui-asset-download.sh'
       - '!shared/bin/os-disk-config.py'
       - '!shared/bin/preseed_late_user_config.sh'
       - '!shared/bin/configure-interfaces.py'
-      - '!shared/bin/configure-capture.py'
       - '!shared/bin/suricata*'
       - '!shared/bin/zeek*'
       - '.trigger_workflow_build'

.github/workflows/logstash-build-and-push-ghcr.yml

@@ -9,16 +9,13 @@ on:
       - 'logstash/**'
       - 'Dockerfiles/logstash.Dockerfile'
       - 'shared/bin/*'
-      - '!shared/bin/agg-init.sh'
       - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
-      - '!shared/bin/sensor-init.sh'
       - '!shared/bin/extracted_files_http_server.py'
       - '!shared/bin/web-ui-asset-download.sh'
       - '!shared/bin/os-disk-config.py'
       - '!shared/bin/preseed_late_user_config.sh'
       - '!shared/bin/configure-interfaces.py'
-      - '!shared/bin/configure-capture.py'
       - '!shared/bin/suricata*'
       - '!shared/bin/zeek*'
       - '.trigger_workflow_build'

.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml

@@ -9,7 +9,6 @@ on:
       - 'malcolm-iso/**'
       - 'shared/bin/*'
       - '!shared/bin/capa-build.sh'
-      - '!shared/bin/configure-capture.py'
       - '!shared/bin/extracted_files_http_server.py'
       - '!shared/bin/web-ui-asset-download.sh'
       - '!shared/bin/zeek*'

.github/workflows/netbox-build-and-push-ghcr.yml

@@ -9,16 +9,13 @@ on:
       - 'netbox/**'
       - 'Dockerfiles/netbox.Dockerfile'
       - 'shared/bin/*'
-      - '!shared/bin/agg-init.sh'
       - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
-      - '!shared/bin/sensor-init.sh'
       - '!shared/bin/extracted_files_http_server.py'
       - '!shared/bin/web-ui-asset-download.sh'
       - '!shared/bin/os-disk-config.py'
       - '!shared/bin/preseed_late_user_config.sh'
       - '!shared/bin/configure-interfaces.py'
-      - '!shared/bin/configure-capture.py'
       - '!shared/bin/zeek*'
       - '!shared/bin/suricata*'
       - '.trigger_workflow_build'

.github/workflows/nginx-build-and-push-ghcr.yml

@@ -9,16 +9,13 @@ on:
       - 'nginx/**'
       - 'Dockerfiles/nginx.Dockerfile'
       - 'shared/bin/*'
-      - '!shared/bin/agg-init.sh'
       - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
-      - '!shared/bin/sensor-init.sh'
       - '!shared/bin/extracted_files_http_server.py'
       - '!shared/bin/web-ui-asset-download.sh'
       - '!shared/bin/os-disk-config.py'
       - '!shared/bin/preseed_late_user_config.sh'
       - '!shared/bin/configure-interfaces.py'
-      - '!shared/bin/configure-capture.py'
       - '!shared/bin/zeek*'
       - '!shared/bin/suricata*'
       - '.trigger_workflow_build'

.github/workflows/opensearch-build-and-push-ghcr.yml

@@ -8,16 +8,13 @@ on:
     paths:
       - 'Dockerfiles/opensearch.Dockerfile'
       - 'shared/bin/*'
-      - '!shared/bin/agg-init.sh'
       - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
-      - '!shared/bin/sensor-init.sh'
       - '!shared/bin/os-disk-config.py'
       - '!shared/bin/extracted_files_http_server.py'
       - '!shared/bin/web-ui-asset-download.sh'
       - '!shared/bin/preseed_late_user_config.sh'
       - '!shared/bin/configure-interfaces.py'
-      - '!shared/bin/configure-capture.py'
       - '!shared/bin/zeek*'
       - '!shared/bin/suricata*'
       - '.trigger_workflow_build'

.github/workflows/pcap-capture-build-and-push-ghcr.yml

@@ -9,16 +9,13 @@ on:
       - 'pcap-capture/**'
       - 'Dockerfiles/pcap-capture.Dockerfile'
       - 'shared/bin/*'
-      - '!shared/bin/agg-init.sh'
       - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
-      - '!shared/bin/sensor-init.sh'
       - '!shared/bin/os-disk-config.py'
       - '!shared/bin/extracted_files_http_server.py'
       - '!shared/bin/web-ui-asset-download.sh'
       - '!shared/bin/preseed_late_user_config.sh'
       - '!shared/bin/configure-interfaces.py'
-      - '!shared/bin/configure-capture.py'
       - '!shared/bin/zeek*'
       - '!shared/bin/suricata*'
       - '.trigger_workflow_build'

.github/workflows/pcap-monitor-build-and-push-ghcr.yml

@@ -9,16 +9,13 @@ on:
       - 'pcap-monitor/**'
       - 'Dockerfiles/pcap-monitor.Dockerfile'
       - 'shared/bin/*'
-      - '!shared/bin/agg-init.sh'
       - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
-      - '!shared/bin/sensor-init.sh'
       - '!shared/bin/os-disk-config.py'
       - '!shared/bin/extracted_files_http_server.py'
       - '!shared/bin/web-ui-asset-download.sh'
       - '!shared/bin/preseed_late_user_config.sh'
       - '!shared/bin/configure-interfaces.py'
-      - '!shared/bin/configure-capture.py'
       - '!shared/bin/zeek*'
       - '!shared/bin/suricata*'
       - '.trigger_workflow_build'

.github/workflows/postgresql-build-and-push-ghcr.yml

@@ -8,16 +8,13 @@ on:
     paths:
       - 'Dockerfiles/postgresql.Dockerfile'
       - 'shared/bin/*'
-      - '!shared/bin/agg-init.sh'
       - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
-      - '!shared/bin/sensor-init.sh'
       - '!shared/bin/os-disk-config.py'
       - '!shared/bin/extracted_files_http_server.py'
       - '!shared/bin/web-ui-asset-download.sh'
       - '!shared/bin/preseed_late_user_config.sh'
       - '!shared/bin/configure-interfaces.py'
-      - '!shared/bin/configure-capture.py'
       - '!shared/bin/zeek*'
       - '!shared/bin/suricata*'
       - '.trigger_workflow_build'

.github/workflows/redis-build-and-push-ghcr.yml

@@ -8,16 +8,13 @@ on:
     paths:
       - 'Dockerfiles/redis.Dockerfile'
       - 'shared/bin/*'
-      - '!shared/bin/agg-init.sh'
       - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
-      - '!shared/bin/sensor-init.sh'
       - '!shared/bin/os-disk-config.py'
       - '!shared/bin/extracted_files_http_server.py'
       - '!shared/bin/web-ui-asset-download.sh'
       - '!shared/bin/preseed_late_user_config.sh'
       - '!shared/bin/configure-interfaces.py'
-      - '!shared/bin/configure-capture.py'
       - '!shared/bin/zeek*'
       - '!shared/bin/suricata*'
       - '.trigger_workflow_build'

.github/workflows/suricata-build-and-push-ghcr.yml

@@ -9,16 +9,13 @@ on:
       - 'suricata/**'
       - 'Dockerfiles/suricata.Dockerfile'
       - 'shared/bin/*'
-      - '!shared/bin/agg-init.sh'
       - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
-      - '!shared/bin/sensor-init.sh'
       - '!shared/bin/os-disk-config.py'
       - '!shared/bin/extracted_files_http_server.py'
       - '!shared/bin/web-ui-asset-download.sh'
       - '!shared/bin/preseed_late_user_config.sh'
       - '!shared/bin/configure-interfaces.py'
-      - '!shared/bin/configure-capture.py'
       - '!shared/bin/zeek*'
       - '.trigger_workflow_build'
   workflow_dispatch:

.github/workflows/zeek-build-and-push-ghcr.yml

@@ -9,16 +9,13 @@ on:
       - 'zeek/**'
       - 'Dockerfiles/zeek.Dockerfile'
       - 'shared/bin/*'
-      - '!shared/bin/agg-init.sh'
       - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
-      - '!shared/bin/sensor-init.sh'
       - '!shared/bin/os-disk-config.py'
       - '!shared/bin/extracted_files_http_server.py'
       - '!shared/bin/web-ui-asset-download.sh'
       - '!shared/bin/preseed_late_user_config.sh'
       - '!shared/bin/configure-interfaces.py'
-      - '!shared/bin/configure-capture.py'
       - '!shared/bin/suricata*'
       - '.trigger_workflow_build'
   workflow_dispatch:

Dockerfiles/dashboards-helper.Dockerfile

@@ -60,8 +60,6 @@ COPY --chmod=755 shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 COPY --chmod=755 shared/bin/service_check_passthrough.sh /usr/local/bin/
 COPY --from=ghcr.io/mmguero-dev/gostatic --chmod=755 /goStatic /usr/bin/goStatic
 COPY --chmod=755 shared/bin/opensearch_status.sh /data/
-COPY --chmod=755 shared/bin/opensearch_index_size_prune.py /data/
-COPY --chmod=755 shared/bin/opensearch_read_only.py /data/
 ADD scripts/malcolm_utils.py /data/
 
 RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') && \

Dockerfiles/logstash.Dockerfile

@@ -83,7 +83,6 @@ COPY --chmod=755 shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 COPY --chmod=755 shared/bin/service_check_passthrough.sh /usr/local/bin/
 COPY --chmod=755 shared/bin/opensearch_status.sh /usr/local/bin/
 COPY --from=ghcr.io/mmguero-dev/gostatic --chmod=755 /goStatic /usr/bin/goStatic
-COPY --chmod=755 shared/bin/manuf-oui-parse.py /usr/local/bin/
 COPY --chmod=755 shared/bin/jdk-cacerts-auto-import.sh /usr/local/bin/
 COPY --chmod=755 shared/bin/keystore-bootstrap.sh /usr/local/bin/
 ADD logstash/maps/*.yaml /etc/

docs/malcolm-config.md

@@ -175,6 +175,6 @@ In instances where Malcolm is deployed with the intention of running indefinitel
     - `EXTRACTED_FILE_PRUNE_THRESHOLD_MAX_SIZE` - specifies the maximum size, specified either in gigabytes or as a human-readable data size (e.g., `250G`), that the  `./zeek-logs/extract_files/` directory is allowed to contain before the prune condition triggers
     - `EXTRACTED_FILE_PRUNE_THRESHOLD_TOTAL_DISK_USAGE_PERCENT` - specifies a maximum fill percentage for the file system containing the `./zeek-logs/extract_files/`; in other words, if the disk is more than this percentage utilized, the prune condition triggers
     - `EXTRACTED_FILE_PRUNE_INTERVAL_SECONDS` - the interval between checking the prune conditions, in seconds (default `300`)
-* [Index management policies](index-management.md) can be handled via plugins provided as part of the OpenSearch and Elasticsearch platforms, respectively. In addition to those tools, the `OPENSEARCH_INDEX_SIZE_PRUNE_LIMIT` variable in **`dashboards-helper.env`** defines a maximum cumulative that OpenSearch indices are allowed to consume before the oldest indices [are deleted]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/opensearch_index_size_prune.py), specified as either as a human-readable data size (e.g., `250G`) or as a percentage of the total disk size (e.g., `70%`): e.g., a value of `500G` means "delete the oldest OpenSearch indices if the total space consumed by Malcolm's indices exceeds five hundred gigabytes."
+* [Index management policies](index-management.md) can be handled via plugins provided as part of the OpenSearch and Elasticsearch platforms, respectively. In addition to those tools, the `OPENSEARCH_INDEX_SIZE_PRUNE_LIMIT` variable in **`dashboards-helper.env`** defines a maximum cumulative that OpenSearch indices are allowed to consume before the oldest indices [are deleted]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/dashboards/scripts/opensearch_index_size_prune.py), specified as either as a human-readable data size (e.g., `250G`) or as a percentage of the total disk size (e.g., `70%`): e.g., a value of `500G` means "delete the oldest OpenSearch indices if the total space consumed by Malcolm's indices exceeds five hundred gigabytes."
 
 Similar settings exist for managing disk usage on [Hedgehog Linux](malcolm-hedgehog-e2e-iso-install.md#HedgehogDiskUsage).