apps sc & wc: Updating KubeContainerOOMKilled threshold

elastisys · Sep 6, 2023 · 7267299 · 7267299
1 parent 4836dfb
commit 7267299
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 1 deletion.
diff --git a/WIP-CHANGELOG.md b/WIP-CHANGELOG.md
@@ -0,0 +1,13 @@
+### Release notes
+
+### Added
+
+### Changed
+
+- Changed the alert `KubeContainerOOMKilled` threshold.
+
+### Fixed
+
+### Updated
+
+### Removed
diff --git a/helmfile/charts/prometheus-alerts/templates/alerts/kubernetes-apps.yaml b/helmfile/charts/prometheus-alerts/templates/alerts/kubernetes-apps.yaml
@@ -53,7 +53,7 @@ spec:
  annotations:
  description: Container {{`{{`}} $labels.container {{`}}`}} in pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} from {{`{{`}} $labels.cluster {{`}}`}} has been OOMKilled {{`{{`}} $value {{`}}`}} times in the last 10 minutes.
  summary: Kubernetes container OOMKilled.
- expr: (kube_pod_container_status_restarts_total - kube_pod_container_status_restarts_total offset 10m >= 1) and ignoring (reason) min_over_time(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[10m]) == 1
+ expr: (kube_pod_container_status_restarts_total - kube_pod_container_status_restarts_total offset 30m >= 2) and ignoring (reason) min_over_time(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[30m]) == 1
  for: 0m
  labels:
  severity: warning