apps sc: thanos receiver out-of-order bug fix

elastisys · Sep 19, 2023 · dc472ae · dc472ae
1 parent ffb1a0f
commit dc472ae
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 2 deletions.
diff --git a/WIP-CHANGELOG.md b/WIP-CHANGELOG.md
@@ -16,6 +16,7 @@
 ### Fixed
 
 - Broken link in v0.30 migration instructions
+- Fixed thanos ingesting out-of-order error.
 
 ### Updated
 

diff --git a/config/config/sc-config.yaml b/config/config/sc-config.yaml
@@ -601,6 +601,9 @@ thanos:
  # standalone: Single instance of receiver.
  mode: dual-mode
 
+ # Can only be used if thanos.compactor.verticalCompaction is set to true
+ outOfOrderTimeWindow: 600s
+
  # Retention for the metrics in the receiver
  tsdbRetention: 15d
 
@@ -630,6 +633,8 @@ thanos:
  retentionResolution5m: 90d
  retentionResolution1h: 0s
 
+ verticalCompaction: false
+
  # Deduplication of metrics in long term storage
  # receiverReplicas: Deduplicate results from multiple receivers, simpler dedup
  # prometheusReplicas: Deduplicate results from multiple receivers and multiple prometheis, heavier dedup

diff --git a/helmfile/values/thanos/receiver.yaml.gotmpl b/helmfile/values/thanos/receiver.yaml.gotmpl
@@ -13,12 +13,14 @@ compactor:
  retentionResolution5m: {{ .Values.thanos.compactor.retentionResolution5m }}
  retentionResolution1h: {{ .Values.thanos.compactor.retentionResolution1h }}
 
- {{- if eq .Values.thanos.compactor.deduplication "receiverReplicas" }}
  extraFlags:
+ {{- if eq .Values.thanos.compactor.verticalCompaction true }}
+ - --compact.enable-vertical-compaction
+ {{- end }}
+ {{- if eq .Values.thanos.compactor.deduplication "receiverReplicas" }}
  - --deduplication.func=
  - --deduplication.replica-label=replica
  {{- else if eq .Values.thanos.compactor.deduplication "prometheusReplicas" }}
- extraFlags:
  - --deduplication.func=penalty
  - --deduplication.replica-label=replica
  - --deduplication.replica-label=prometheus_replica
@@ -44,6 +46,10 @@ receive:
 
  mode: {{ .Values.thanos.receiver.mode }}
 
+ {{- if eq .Values.thanos.compactor.verticalCompaction true }}
+ extraFlags:
+ - --tsdb.out-of-order.time-window={{ .Values.thanos.receiver.outOfOrderTimeWindow }}
+ {{- end }}
  # extraFlags:
  # - "--receive.tenant-label-name=\"cluster\""