Stop channel bug fix and reload frequency (#20425)

Unreleased bug fix for #19352

(cherry picked from commit dc7a3cd)

ee/server/calendar/google_calendar.go

@@ -270,19 +270,14 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn
 		return nil, false, err
 	}
 	gEvent, err := c.config.API.GetEvent(details.ID, details.ETag)
-	var deleted bool
+	var deleted, channelStopped bool
 	switch {
 	// http.StatusNotModified is returned sometimes, but not always, so we need to check ETag explicitly later
 	case googleapi.IsNotModified(err):
 		return event, false, nil
 	// http.StatusNotFound should be very rare -- Google keeps events for a while after they are deleted
 	case isNotFound(err):
 		deleted = true
-		// If event was deleted, we need to stop watching it
-		err = c.config.API.Stop(details.ChannelID, details.ResourceID)
-		if err != nil {
-			level.Warn(c.config.Logger).Log("msg", "stopping Google calendar event watch", "err", err)
-		}
 	case err != nil:
 		return nil, false, ctxerr.Wrap(c.config.Context, err, "retrieving Google calendar event")
 	}
@@ -304,6 +299,7 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn
 				level.Warn(c.config.Logger).Log("msg", "deleting Google calendar event which was changed to all-day event", "err", err)
 			}
 			deleted = true
+			channelStopped = true
 		}
 
 		var endTime *time.Time
@@ -320,6 +316,7 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn
 					level.Warn(c.config.Logger).Log("msg", "deleting Google calendar event which is in the past", "err", err)
 				}
 				deleted = true
+				channelStopped = true
 			}
 		}
 		if !deleted {
@@ -335,6 +332,7 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn
 					level.Warn(c.config.Logger).Log("msg", "deleting Google calendar event which was changed to all-day event", "err", err)
 				}
 				deleted = true
+				channelStopped = true
 			}
 		}
 		if !deleted {
@@ -350,6 +348,14 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn
 		}
 	}
 
+	// If event was deleted/cancelled, we need to stop watching it
+	if !channelStopped {
+		err = c.config.API.Stop(details.ChannelID, details.ResourceID)
+		if err != nil {
+			level.Warn(c.config.Logger).Log("msg", "stopping Google calendar event watch", "err", err)
+		}
+	}
+
 	newStartDate := calculateNewEventDate(event.StartTime)
 
 	fleetEvent, err := c.CreateEvent(newStartDate, genBodyFn)

server/cron/calendar_cron.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/google/uuid"
 	"slices"
 	"sync"
 	"time"
@@ -17,11 +16,12 @@ import (
 	"github.com/go-kit/log"
 	kitlog "github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/google/uuid"
 )
 
 const (
 	calendarConsumers = 18
-	reloadFrequency   = 12 * time.Hour
+	reloadFrequency   = 30 * time.Minute
 )
 
 func NewCalendarSchedule(
@@ -385,13 +385,6 @@ func processFailingHostExistingCalendarEvent(
 		}
 	}()
 
-	// Remove event from the queue so that we don't process this event again.
-	// Note: This item can be added back to the queue while we are processing it.
-	err = distributedLock.RemoveFromSet(ctx, calendar.QueueKey, eventUUID)
-	if err != nil {
-		return fmt.Errorf("remove calendar event from queue: %w", err)
-	}
-
 	updatedEvent := calendarEvent
 	updated := false
 	now := time.Now()
@@ -422,6 +415,12 @@ func processFailingHostExistingCalendarEvent(
 		); err != nil {
 			return fmt.Errorf("updating event calendar on db: %w", err)
 		}
+
+		// Remove event from the queue so that we don't process this event again.
+		err = distributedLock.RemoveFromSet(ctx, calendar.QueueKey, eventUUID)
+		if err != nil {
+			return fmt.Errorf("remove calendar event from queue: %w", err)
+		}
 	}
 
 	eventInFuture := now.Before(updatedEvent.StartTime)