monitoring-client: add to-disk logging of Bitswap messages

Dockerfile.bitswap-monitoring-client

@@ -1,36 +1,65 @@
 # Implements an image to run the bitswap-monitoring-client tool.
 # This will expose port 8088 for prometheus.
-# The executable is placed in /, the config in /config/.
+# The executable is placed in /ipfs-tools, the config in /ipfs-tools/config/.
 # The config is copied from the builder stage (and thus verbose from the sources).
-# You can probably overwrite it by mounting your own config directory, I guess.
+# You can override it by mounting your own.
+
+# First build su-exec
+FROM ubuntu:jammy AS builder
+
+RUN apt-get update && apt-get install -y \
+ curl \
+ build-essential \
+ git \
+ wget
+
+# Get su-exec, a very minimal tool for dropping privileges.
+ENV SUEXEC_VERSION=v0.2
+RUN set -eux; \
+ dpkgArch="$(dpkg --print-architecture)"; \
+ case "${dpkgArch##*-}" in \
+ "amd64" | "armhf" | "arm64") tiniArch="tini-static-$dpkgArch" ;;\
+ *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \
+ esac; \
+ cd /tmp \
+ && git clone https://github.com/ncopa/su-exec.git \
+ && cd su-exec \
+ && git checkout -q $SUEXEC_VERSION \
+ && make su-exec-static
+
+# Get yq
+ENV YQ_VERSION=v4.44.3
+RUN set -eux; \
+ dpkgArch="$(dpkg --print-architecture)"; \
+ case "${dpkgArch##*-}" in \
+ "amd64" | "arm" | "armhf" | "arm64") tiniArch="tini-static-$dpkgArch" ;;\
+ *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \
+ esac; \
+ wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${dpkgArch} -O /usr/bin/yq &&\
+ chmod +x /usr/bin/yq
 
 # Get some small base image to run things on.
 FROM ubuntu:jammy AS runtime
 
-# Create a system user to drop into.
-# This will get some small (<1000) UID and GID, which is fine since we don't write to any files on the host.
-RUN groupadd -r ipfs \
- && useradd --no-log-init -r -g ipfs ipfs \
- && mkdir -p ipfs
-
 # Enter our working directory.
 WORKDIR ipfs-tools
 
 # Copy compiled binaries from builder.
 COPY --from=ipfs-tools-builder /ipfs-tools/target/release/bitswap-monitoring-client .
 COPY --from=ipfs-tools-builder /ipfs-tools/bitswap-monitoring-client/config.yaml ./config/bitswap-monitoring-client-config.yaml
+COPY --from=ipfs-tools-builder /ipfs-tools/bitswap-monitoring-client/docker-entrypoint.sh .
+COPY --from=0 /tmp/su-exec/su-exec-static /sbin/su-exec
+COPY --from=0 /usr/bin/yq /usr/bin/yq
 
-# Set ownership.
-RUN chown -R ipfs:ipfs ./bitswap-monitoring-client
+# Make sure our entrypoint is executable.
+RUN chmod 755 ./docker-entrypoint.sh
 
 # Set log level.
 ENV RUST_LOG=info
 
 # Expose Prometheus endpoint.
 EXPOSE 8088
 
-# Drop root.
-USER ipfs
-
-# Run the binary.
-ENTRYPOINT ["./bitswap-monitoring-client","--config","./config/bitswap-monitoring-client-config.yaml"]
+# Run the script.
+# This will fix permissions on the temporary file storage directory, drop root, and then run the binary.
+ENTRYPOINT ["./docker-entrypoint.sh"]

bitswap-monitoring-client/Cargo.toml

@@ -7,11 +7,13 @@ edition = "2021"
 [dependencies]
 ipfs-resolver-common = { path = "../common" }
 ipfs_monitoring_plugin_client = { path = "../ipfs-monitoring-plugin-client" }
-tokio = { version = "1", features = ["rt", "net", "sync", "rt-multi-thread", "time", "macros", "signal", "fs"] }
+tokio = { version = "^1.21", features = ["rt", "net", "sync", "rt-multi-thread", "time", "macros", "signal", "fs", "io-util"] }
 log = "0.4.21"
 flexi_logger = "0.28.5"
 failure = "0.1.8"
 futures-util = "0.3.29"
+tokio-util = "0.7.10"
+futures = "0.3.30"
 chrono = { version = "0.4.31", features = ["serde"] }
 prometheus_exporter = "0.8.4"
 # This needs to be matching the version prometheus_exporter uses!
@@ -29,3 +31,7 @@ multiaddr = "0.17"
 
 # ISO 3166-1 countries.
 celes = "2.4.0"
+
+# Logging to file
+async-compression = { version = "0.3.15" , default-features = false, features=["tokio","gzip"]}
+serde_json = "1.0.96"

bitswap-monitoring-client/README.md

@@ -16,7 +16,7 @@ This is an example config file, see also the [file](./config.yaml) and the [impl
 # This is a config file for the bitswap-monitoring-client tool.
 
 # Address to listen and serve prometheus metrics on.
-prometheus_address: "0.0.0.0:8080"
+prometheus_address: "0.0.0.0:8088"
 
 # Specifies the path to the MaxMind GeoLite databases.
 # Defaults to /usr/local/share/GeoIP if unspecified.
@@ -28,6 +28,11 @@ prometheus_address: "0.0.0.0:8080"
 # Defaults to empty, i.e., no tagging of gateway traffic.
 #gateway_file_path: "/usr/local/share/gateways.txt"
 
+# Specifies a path to a directory to write JSON logs.
+# A subdirectory per monitor will be created.
+# If not provided, logging to disk will be disabled.
+#disk_logging_directory: "traces"
+
 # List of AMQP data sources to connect to.
 amqp_servers:
  # Address of the AMQP server, using amqp or amqps (TLS transport) scheme.
@@ -40,6 +45,20 @@ amqp_servers:
 The `prometheus_address` specifies the local endpoint to listen and serve Prometheus metrics on.
 For each (`amqp_server`, `monitor_name`) combination, a connection to the AMQP server will be opened.
 
+### Docker
+
+When running in docker via [../Dockerfile.bitswap-monitoring-client](../Dockerfile.bitswap-monitoring-client),
+the client is started via the [docker-entrypoint.sh](docker-entrypoint.sh) script.
+This looks for the environment variables `PUID` and `PGID`, `chown`s the logging directory, and drops root for the
+given UID and GID.
+
+## To-Disk Logging
+
+If enabled via `disk_logging_directory`, the client writes logs as gzipped JSON files into the configured directory.
+A subdirectory per monitor will be created.
+Log files are rotated hourly.
+The client listens for `SIGINT` and `SIGTERM` to shut down, and finalizes the currently-opened file.
+
 ## Metrics
 
 Metrics are provided via a Prometheus HTTP endpoint.

bitswap-monitoring-client/config.yaml

@@ -13,10 +13,15 @@ prometheus_address: "0.0.0.0:8088"
 # Defaults to empty, i.e., no tagging of gateway traffic.
 #gateway_file_path: "/usr/local/share/gateways.txt"
 
+# Specifies a path to a directory to write JSON logs.
+# A subdirectory per monitor will be created.
+# If not provided, logging to disk will be disabled.
+#disk_logging_directory: "traces"
+
 # List of AMQP data sources to connect to.
 amqp_servers:
  # Address of the AMQP server, using amqp or amqps (TLS transport) scheme.
  - amqp_server_address: "amqp://localhost:5672/%2f"
  # A list of monitors to subscribe to via this data source.
  monitor_names:
- - "local"
+ - "local"

bitswap-monitoring-client/docker-entrypoint.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -ex
+
+user_id=$PUID
+user_gid=$PGID
+if [ -z "$PUID" ]; then
+ echo "PUID unset, using default value of 1000"
+ user_id=1000
+fi
+if [ -z "$PGID" ]; then
+ echo "PGID unset, using default value of 1000"
+ user_gid=1000
+fi
+
+traces_dir=$(yq '.disk_logging_directory' ./config/bitswap-monitoring-client-config.yaml)
+
+if [ "$(id -u)" -eq 0 ]; then
+ echo "Changing user to $user_id"
+ if [ ! "$traces_dir" == "null" ]; then
+ echo "Fixing permissions on logging directory $traces_dir..."
+ # ensure traces directory is writable
+ su-exec "$user_id" test -w "$traces_dir" || chown -R -- "$user_id:$user_gid" "$traces_dir"
+ fi
+ # restart script with new privileges
+ exec su-exec "$user_id:$user_gid" "$0" "$@"
+fi
+
+# 2nd invocation with regular user
+exec ./bitswap-monitoring-client "$@"

bitswap-monitoring-client/src/config.rs

@@ -23,6 +23,11 @@ pub(crate) struct Config {
  /// Each line in the file should contain one peer ID.
  /// If not provided, all traffic will be logged as non-gateway traffic.
  pub(crate) gateway_file_path: Option<String>,
+
+ /// Specifies a path to a directory to write bitswap traces to.
+ /// A subdirectory per monitor will be created.
+ /// If not provided, logging to disk will be disabled.
+ pub(crate) disk_logging_directory: Option<String>,
 }
 
 /// Configuration for a single data source.