-
Notifications
You must be signed in to change notification settings - Fork 0
/
statsd_mapping.yml
100 lines (87 loc) · 3.29 KB
/
statsd_mapping.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
# see https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/logging-monitoring/metrics.html#metrics
mappings:
- match: "(.+)\\.operator_successes_(.+)$"
match_metric_type: counter
name: "af_agg_operator_successes"
# confirmed
match_type: regex
labels:
airflow_id: "$1"
operator_name: "$2"
- match: "*.ti_failures"
match_metric_type: counter
name: "af_agg_ti_failures"
# confirmed
labels:
airflow_id: "$1"
- match: "*.ti_successes"
match_metric_type: counter
name: "af_agg_ti_successes"
# confirmed
labels:
airflow_id: "$1"
- match: "airflow.executor.open_slots"
help: The number of open executor slots
name: "airflow_executor_open_slots"
# confirmed
metric_type: gauge
- match: "airflow.executor.queued_tasks"
help: The number of queued tasks
name: "airflow_executor_queued_tasks"
metric_type: gauge
- match: "airflow.executor.running_tasks"
help: The number of running tasks
name: "airflow_executor_running_tasks"
metric_type: gauge
- match: "(.*)ti(.+)scheduled$"
name: "af_agg_ti_scheduled"
match_type: regex
# example airflow_ti_finish_wiley_ltd_full_Step_1_scheduled
# this one doesn't seem to be working, not present
help: Aggregated number of tasks scheduled
match_metric_type: gauge
labels:
prefix: "$1"
airflow_id: "$2"
- match: "airflow\\.ti\\.(.+)\\.queued$"
name: "af_agg_ti_queued"
match_type: regex
# example airflow_ti_finish_meyer_plc_full_Step_1_queued
# airflow_ti_finish_wiley_ltd_full_Step_1_queued
# not found
help: Aggregated number of tasks queued
match_metric_type: gauge
labels:
airflow_id: "$1"
- match: "airflow\\.scheduler\\.tasks\\.(.*)\\.running$"
name: "af_agg_ti_running"
match_type: regex
# example airflow_ti_finish_meyer_plc_full_Step_1_running
help: Aggregated number of tasks running
match_metric_type: gauge
labels:
prefix: "$1"
airflow_id: "$2"
- match: "(.+)deferred$"
name: "af_agg_ti_deferred"
match_type: regex
# example airflow_ti_finish_meyer_plc_full_Step_1_deferred
# airflow_ti_finish_bailey_llc_full_Step_1_deferred 16
help: Aggregated number of tasks deferred
match_metric_type: gauge
labels:
airflow_id: "$1"
- match: "airflow.scheduler.tasks.starving"
help: Number of tasks that cannot be scheduled because of no open slot in pool
name: "airflow_scheduler_tasks_starving"
metric_type: gauge
- match: "scheduler.tasks.executable"
help: Number of tasks that are ready for execution (set to queued) with respect to pool limits, DAG concurrency, executor state, and priority.
name: "airflow_scheduler_tasks_executable"
metric_type: gauge
# from airflow's exporter
# airflow_dag_status{dag_id="bailey-and-sons-incremental",owner="airflow",status="queued"} 68.0
# airflow_dag_status{dag_id="bailey-and-sons-incremental",owner="airflow",status="running"} 16.0
# airflow_dag_status{dag_id="bailey-and-sons-incremental",owner="airflow",status="success"} 16.0
# airflow_dag_status{dag_id="bailey-and-sons-incremental",owner="airflow",status="failed"} 0.0
# airflow_dag_status{dag_id="bailey-and-sons-incremental",owner="airflow",status="none"} 0.0