Add ray-logs-sidecar

.github/workflows/ray-logs-sidecar.yml

@@ -0,0 +1,45 @@
+name: Build & Push Ray Logs Sidecar Image
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+on:
+ pull_request:
+ paths:
+ - .github/workflows/ray-logs-sidecar.yml
+ - ray-logs-sidecar/**
+ push:
+ branches:
+ - master
+ workflow_dispatch:
+
+jobs:
+ build-and-push-image:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ container: [ray-logs-sidecar]
+ steps:
+ - uses: actions/checkout@v4
+ - uses: docker/setup-qemu-action@v3
+ - uses: docker/setup-buildx-action@v3
+ - uses: docker/login-action@v2
+ if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
+ with:
+ registry: https://ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+ - id: image-names
+ uses: docker/metadata-action@v3
+ with:
+ images: ghcr.io/${{ github.repository_owner }}/${{ matrix.container }}
+ tags: |
+ type=raw,value=latest,enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+ type=sha,format=long
+ - uses: docker/build-push-action@v5
+ with:
+ context: ${{ matrix.container }}
+ platforms: linux/amd64,linux/arm64
+ push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
+ tags: ${{ steps.image-names.outputs.tags }}

README.md

@@ -1 +1,10 @@
-# containers
+# Containers
+
+This repository contains Dockerfiles for useful containers that can be used with [Flyte](https://github.com/flyteorg/flyte).
+
+The following containers are currently available:
+
+| Container | Description |
+| ------------------ | ------------------------------------------------------- |
+| `ray-logs-sidecar` | Captures and exposes Ray job logs to container's stdout |
+

ray-logs-sidecar/Dockerfile

@@ -0,0 +1,3 @@
+FROM fluent/fluent-bit:2.1.10
+LABEL org.opencontainers.image.source="https://github.com/unionai-oss/containers"
+COPY fluent-bit.conf strip-ansi-colors.lua /fluent-bit/etc/

ray-logs-sidecar/fluent-bit.conf

@@ -0,0 +1,17 @@
+[SERVICE]
+ Log_Level error
+[INPUT]
+ Name tail
+ Path /tmp/ray/session_latest/logs/job-driver-*
+ Refresh_Interval 1
+[FILTER]
+ Name lua
+ Match *
+ script strip-ansi-colors.lua
+ call strip_ansi_colors
+[OUTPUT]
+ Name file
+ Match *
+ File /dev/stdout
+ Format template
+ Template {log}

ray-logs-sidecar/strip-ansi-colors.lua

@@ -0,0 +1,11 @@
+-- From: https://github.com/fluent/fluent-bit/discussions/6151#discussioncomment-4392962
+function strip_ansi_colors(tag, timestamp, record)
+ if type(record["log"]) == "string" then
+ -- strip ansi https://stackoverflow.com/a/49209650/368691
+ record["log"] = record["log"]:gsub('[\27\155][][()#;?%d]*[A-PRZcf-ntqry=><~]', '')
+
+ return 2, 0, record
+ end
+
+ return 0, 0, 0
+end