fix: delayed detection of job ending for crons

crons.go

@@ -36,38 +36,38 @@ func startCronsInformers(ctx context.Context, namespace string) error {
 		return errors.New("failed to get clientset")
 	}
 
-	// create factory that will produce both the cronjob informer and job informer
+	// Create factory that will produce both the cronjob informer and job informer
 	factory := informers.NewSharedInformerFactoryWithOptions(
 		clientset,
 		5*time.Second,
 		informers.WithNamespace(namespace),
 	)
 
-	// create the cronjob informer
+	// Create the cronjob informer
 	cronjobInformer, err = createCronjobInformer(ctx, factory, namespace)
 	if err != nil {
 		return err
 	}
-	// create the job informer
+	// Create the job informer
 	jobInformer, err = createJobInformer(ctx, factory, namespace)
 	if err != nil {
 		return err
 	}
 
-	// channel to tell the factory to stop the informers
+	// Channel to tell the factory to stop the informers
 	doneChan := make(chan struct{})
 	factory.Start(doneChan)
 
-	// sync the cronjob informer cache
+	// Sync the cronjob informer cache
 	if ok := cache.WaitForCacheSync(doneChan, cronjobInformer.HasSynced); !ok {
 		return errors.New("cronjob informer failed to sync")
 	}
-	// sync the job informer cache
+	// Sync the job informer cache
 	if ok := cache.WaitForCacheSync(doneChan, jobInformer.HasSynced); !ok {
 		return errors.New("job informer failed to sync")
 	}
 
-	// wait for the channel to be closed
+	// Wait for the channel to be closed
 	<-doneChan
 
 	return nil
@@ -91,19 +91,20 @@ func runSentryCronsCheckin(ctx context.Context, job *batchv1.Job, eventHandlerTy
 		return errors.New("cannot find cronJob data")
 	}
 
-	// capture checkin event called for by informer handler
-	if eventHandlerType == EventHandlerAdd {
+	// The job just begun so check in to start
+	if job.Status.Active == 0 && job.Status.Succeeded == 0 && job.Status.Failed == 0 {
 		// Add the job to the cronJob informer data
 		checkinJobStarting(ctx, job, cronsMonitorData)
-	} else if eventHandlerType == EventHandlerUpdate || eventHandlerType == EventHandlerDelete {
-		// Delete pod from the cronJob informer data
+	} else if job.Status.Active > 0 {
+		return nil
+	} else if job.Status.Failed > 0 || job.Status.Succeeded > 0 {
 		checkinJobEnding(ctx, job, cronsMonitorData)
+		return nil // Job ran successfully
 	}
-
 	return nil
 }
 
-// sends the checkin event to sentry crons for when a job starts
+// Sends the checkin event to sentry crons for when a job starts
 func checkinJobStarting(ctx context.Context, job *batchv1.Job, cronsMonitorData *CronsMonitorData) error {
 
 	logger := zerolog.Ctx(ctx)
@@ -128,21 +129,24 @@ func checkinJobStarting(ctx context.Context, job *batchv1.Job, cronsMonitorData
 	return nil
 }
 
-// sends the checkin event to sentry crons for when a job ends
+// Sends the checkin event to sentry crons for when a job ends
 func checkinJobEnding(ctx context.Context, job *batchv1.Job, cronsMonitorData *CronsMonitorData) error {
 
 	logger := zerolog.Ctx(ctx)
-	// do not check in to exit if there are still active pods
-	if job.Status.Active > 0 {
-		return nil
-	}
 
 	// Check desired number of pods have succeeded
 	var jobStatus sentry.CheckInStatus
-	if job.Status.Succeeded >= cronsMonitorData.requiredCompletions {
-		jobStatus = sentry.CheckInStatusOK
+
+	if job.Status.Conditions == nil {
+		return nil
 	} else {
-		jobStatus = sentry.CheckInStatusError
+		if job.Status.Conditions[0].Type == "Complete" {
+			jobStatus = sentry.CheckInStatusOK
+		} else if job.Status.Conditions[0].Type == "Failed" {
+			jobStatus = sentry.CheckInStatusError
+		} else {
+			return nil
+		}
 	}
 
 	// Get job data to retrieve the checkin ID
@@ -163,7 +167,7 @@ func checkinJobEnding(ctx context.Context, job *batchv1.Job, cronsMonitorData *C
 	return nil
 }
 
-// adds to the sentry events whenever it is associated with a cronjob
+// Adds to the sentry events whenever it is associated with a cronjob
 // so the sentry event contains the corresponding slug monitor, cronjob name, timestamp of when the cronjob began, and
 // the k8s cronjob metadata
 func runCronsDataHandler(ctx context.Context, scope *sentry.Scope, pod *v1.Pod, sentryEvent *sentry.Event) (bool, error) {

informer_crons.go

@@ -10,6 +10,9 @@ import (
 	"k8s.io/client-go/tools/cache"
 )
 
+var checkinMargin int64 = 1
+var maxRunTimeSlack int64 = 3
+
 func createCronjobInformer(ctx context.Context, factory informers.SharedInformerFactory, namespace string) (cache.SharedIndexInformer, error) {
 
 	logger := zerolog.Ctx(ctx)
@@ -27,7 +30,7 @@ func createCronjobInformer(ctx context.Context, factory informers.SharedInformer
 		if ok {
 			logger.Debug().Msgf("cronJob %s already exists in the crons informer data struct...\n", cronjob.Name)
 		} else {
-			cronsMetaData.addCronsMonitorData(cronjob.Name, NewCronsMonitorData(cronjob.Name, cronjob.Spec.Schedule, 5, 3, cronjob.Spec.JobTemplate.Spec.Completions))
+			cronsMetaData.addCronsMonitorData(cronjob.Name, NewCronsMonitorData(cronjob.Name, cronjob.Spec.Schedule, checkinMargin, maxRunTimeSlack, cronjob.Spec.JobTemplate.Spec.Completions))
 		}
 	}