-
Notifications
You must be signed in to change notification settings - Fork 64
/
crons.go
148 lines (127 loc) · 3.92 KB
/
crons.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
package main
import (
"context"
"errors"
"github.com/getsentry/sentry-go"
"github.com/rs/zerolog"
batchv1 "k8s.io/api/batch/v1"
)
type EventHandlerType string
const (
EventHandlerAdd EventHandlerType = "ADD"
EventHandlerUpdate EventHandlerType = "UPDATE"
EventHandlerDelete EventHandlerType = "DELETE"
)
// Captures sentry crons checkin event if appropriate
// by checking the job status to determine if the job just created pod (job starting)
// or if the job exited
func runSentryCronsCheckin(ctx context.Context, job *batchv1.Job, eventHandlerType EventHandlerType) error {
hub := sentry.GetHubFromContext(ctx)
if hub == nil {
return errors.New("cannot get hub from context")
}
// To avoid concurrency issue
hub = hub.Clone()
// Try to find the cronJob name that owns the job
// in order to get the crons monitor data
if len(job.OwnerReferences) == 0 {
return errors.New("job does not have cronjob reference")
}
cronjobRef := job.OwnerReferences[0]
if cronjobRef.Controller == nil || !*cronjobRef.Controller || cronjobRef.Kind != KindCronjob {
return errors.New("job does not have cronjob reference")
}
cronsMonitorData, ok := cronsMetaData.getCronsMonitorData(cronjobRef.Name)
if !ok {
return errors.New("cannot find cronJob data")
}
hub.WithScope(func(scope *sentry.Scope) {
// If DSN annotation provided, we bind a new client with that DSN
client, ok := dsnClientMapping.GetClientFromObject(ctx, &job.ObjectMeta, hub.Client().Options())
if ok {
hub.BindClient(client)
}
// Pass clone hub down with context
ctx = sentry.SetHubOnContext(ctx, hub)
// The job just begun so check in to start
if job.Status.Active == 0 && job.Status.Succeeded == 0 && job.Status.Failed == 0 {
// Add the job to the cronJob informer data
err := checkinJobStarting(ctx, job, cronsMonitorData)
if err != nil {
return
}
} else if job.Status.Active > 0 {
return
} else if job.Status.Failed > 0 || job.Status.Succeeded > 0 {
err := checkinJobEnding(ctx, job, cronsMonitorData)
if err != nil {
return
}
return // Finished
}
})
return nil
}
// Sends the checkin event to sentry crons for when a job starts
func checkinJobStarting(ctx context.Context, job *batchv1.Job, cronsMonitorData *CronsMonitorData) error {
logger := zerolog.Ctx(ctx)
hub := sentry.GetHubFromContext(ctx)
if hub == nil {
return errors.New("cannot get hub from context")
}
// Check if job already added to jobData slice
_, ok := cronsMonitorData.JobDatas[job.Name]
if ok {
return nil
}
logger.Debug().Msgf("Checking in at start of job: %s\n", job.Name)
// All containers running in the pod
checkinID := hub.CaptureCheckIn(
&sentry.CheckIn{
MonitorSlug: cronsMonitorData.MonitorSlug,
Status: sentry.CheckInStatusInProgress,
},
cronsMonitorData.monitorConfig,
)
err := cronsMonitorData.addJob(job, *checkinID)
if err != nil {
return err
}
return nil
}
// Sends the checkin event to sentry crons for when a job ends
func checkinJobEnding(ctx context.Context, job *batchv1.Job, cronsMonitorData *CronsMonitorData) error {
hub := sentry.GetHubFromContext(ctx)
if hub == nil {
return errors.New("cannot get hub from context")
}
logger := zerolog.Ctx(ctx)
// Check desired number of pods have succeeded
var jobStatus sentry.CheckInStatus
if job.Status.Conditions == nil {
return nil
} else {
if job.Status.Conditions[0].Type == "Complete" {
jobStatus = sentry.CheckInStatusOK
} else if job.Status.Conditions[0].Type == "Failed" {
jobStatus = sentry.CheckInStatusError
} else {
return nil
}
}
// Get job data to retrieve the checkin ID
jobData, ok := cronsMonitorData.JobDatas[job.Name]
if !ok {
return nil
}
logger.Trace().Msgf("checking in at end of job: %s\n", job.Name)
hub.CaptureCheckIn(
&sentry.CheckIn{
ID: jobData.getCheckinID(),
MonitorSlug: cronsMonitorData.MonitorSlug,
Status: jobStatus,
},
cronsMonitorData.monitorConfig,
)
return nil
}