Grafana Tempo [WiP]

Makefile

@@ -513,6 +513,13 @@ stop-dependencies-cdc:
 
 start: start-sqlite
 
+start-otel:
+	@OTEL_TRACES_EXPORTER=otlp OTEL_EXPORTER_OTLP_TRACES_INSECURE=true TEMPORAL_OTEL_DEBUG=true make start-sqlite
+
+start-grafana-tempo:
+	docker compose $(DOCKER_COMPOSE_FILES) up -d tempo
+	printf $(COLOR) "Grafana Tempo available at http://localhost:3000"
+
 start-cass-es: temporal-server
 	./temporal-server --env development-cass-es --allow-no-auth start
 

develop/docker-compose/docker-compose.darwin.yml

@@ -14,6 +14,14 @@ services:
       - "3000:3000"
     networks:
       - temporal-dev-network
+  tempo:
+    depends_on:
+      - grafana
+    ports:
+      - "3200:3200" # tempo
+      - "4317:4317" # otlp grpc
+    networks:
+      - temporal-dev-network
   temporal-ui:
     environment:
       - TEMPORAL_ADDRESS=host.docker.internal:7233

develop/docker-compose/docker-compose.linux.yml

@@ -8,6 +8,10 @@ services:
       - ./prometheus-linux:/etc/prometheus
   grafana:
     network_mode: host
+  tempo:
+    depends_on:
+      - grafana
+    network_mode: host
   temporal-ui:
     environment:
       - TEMPORAL_ADDRESS=localhost:7233

develop/docker-compose/docker-compose.windows.yml

@@ -14,6 +14,14 @@ services:
       - "3000:3000"
     networks:
       - temporal-dev-network
+  tempo:
+    depends_on:
+      - grafana
+    ports:
+      - "3200:3200" # tempo
+      - "4317:4317" # otlp grpc
+    networks:
+      - temporal-dev-network
   temporal-ui:
     environment:
       - TEMPORAL_ADDRESS=host.docker.internal:7233

develop/docker-compose/docker-compose.yml

@@ -77,6 +77,13 @@ services:
       - prometheus
     volumes:
       - ./grafana/provisioning/:/etc/grafana/provisioning/
+  tempo:
+    image: grafana/tempo:2.4.2
+    container_name: temporal-dev-grafana-tempo
+    command:
+      - "-config.file=/etc/tempo.yaml"
+    volumes:
+      - ./grafana/provisioning/tempo/tempo.yaml:/etc/tempo.yaml
   temporal-ui:
     image: temporalio/ui:2.26.2
     container_name: temporal-dev-ui

develop/docker-compose/grafana/provisioning/datasources/prometheus.yml

@@ -9,4 +9,14 @@ datasources:
   type: prometheus
   url: http://prometheus:9090
   access: proxy
+  isDefault: false
+- name: Tempo-Linux
+  type: tempo
+  access: proxy
+  url: http://localhost:3200
+  isDefault: false
+- name: Tempo-Darwin
+  type: tempo
+  access: proxy
+  url: http://tempo:3200
   isDefault: false

develop/docker-compose/grafana/provisioning/tempo/tempo.yaml

@@ -0,0 +1,59 @@
+stream_over_http_enabled: true
+server:
+  http_listen_port: 3200
+  log_level: info
+
+query_frontend:
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 5s
+
+distributor:
+  receivers:                           # this configuration will listen on all ports and protocols that tempo is capable of.
+    jaeger:                            # the receives all come from the OpenTelemetry collector.  more configuration information can
+      protocols:                       # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver
+        thrift_http:                   #
+        grpc:                          # for a production deployment you should only enable the receivers you need!
+        thrift_binary:
+        thrift_compact:
+    zipkin:
+    otlp:
+      protocols:
+        http:
+        grpc:
+    opencensus:
+
+ingester:
+  max_block_duration: 5m               # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
+
+compactor:
+  compaction:
+    block_retention: 1h                # overall Tempo trace retention. set for demo purposes
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+      cluster: docker-compose
+  storage:
+    path: /var/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+  traces_storage:
+    path: /var/tempo/generator/traces
+
+storage:
+  trace:
+    backend: local                     # backend configuration to use
+    wal:
+      path: /var/tempo/wal             # where to store the the wal locally
+    local:
+      path: /var/tempo/blocks
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors: [service-graphs, span-metrics, local-blocks] # enables metrics generator

develop/docker-compose/prometheus-darwin/prometheus.yml

@@ -10,3 +10,7 @@ scrape_configs:
       - targets:
           # port should match value from server config listenAddress.
           - host.docker.internal:8000
+  - job_name: tempo
+    static_configs:
+      - targets:
+          - host.docker.internal:3200

develop/docker-compose/prometheus-linux/prometheus.yml

@@ -10,3 +10,7 @@ scrape_configs:
       - targets:
           # port should match value from server config listenAddress.
           - localhost:8000
+  - job_name: tempo
+    static_configs:
+      - targets:
+          - localhost:3200

docs/development/tracing.md

@@ -2,9 +2,8 @@
 
 The Temporal server supports ability to configure OTEL trace exporters to
 support emitting spans and traces for observability. More specifically, the
-server uses the [Go Open Telemetry
-library](https://github.com/open-telemetry/opentelemetry-go) for instrumentation
-and multi-protocol multi-model telemetry exporting. This document is intended to
+server uses the [Go Open Telemetry library](https://github.com/open-telemetry/opentelemetry-go)
+for instrumentation and multi-protocol multi-model telemetry exporting. This document is intended to
 help developers understand how to configure exporters and instrument their code.
 A full exploration of tracing and telemetry is out of scope of this document and
 the reader is referred to [external reference
@@ -15,16 +14,9 @@ itself](https://github.com/open-telemetry/opentelemetry-specification/blob/main/
 
 ## Quickstart
 
-1. Run [`jaeger-all-in-one`](https://www.jaegertracing.io/download/) (either from a binary, or using the docker image).
-2. View the Jaeger UI at http://localhost:16686/search
-3. Set these environment variables:
-    ```
-    export OTEL_TRACES_EXPORTER=otlp
-    export OTEL_EXPORTER_OTLP_TRACES_INSECURE=true
-    export TEMPORAL_OTEL_DEBUG=true
-    ```
-4. Start the server using `make start`
-5. Refresh the Jaeger UI. You should see traces from Temporal services.
+1. Run `make start-grafana-tempo` (or use a different OTEL tool like Jaeger)
+2. Start the server using `make start-otel`
+3. Visit http://localhost:3000/explore and select "Tempo" from the datasource dropdown.
 
 ## Configuring