feat: make lock ttl configurable

manifests/bucketeer/charts/batch/templates/deployment.yaml

@@ -142,6 +142,8 @@ spec:
  value: "{{ .Values.env.nonPersistentRedis.poolMaxIdle }}"
  - name: BUCKETEER_BATCH_NON_PERSISTENT_REDIS_POOL_MAX_ACTIVE
  value: "{{ .Values.env.nonPersistentRedis.poolMaxActive }}"
+ - name: BUCKETEER_BATCH_EXPERIMENT_LOCK_TTL
+ value: "{{ .Values.env.experimentLockTTL }}"
 
  volumeMounts:
  - name: service-cert-secret

manifests/bucketeer/charts/batch/values.yaml

@@ -44,6 +44,7 @@ env:
  addr:
  poolMaxIdle: 25
  poolMaxActive: 25
+ experimentLockTTL: 10m
 
 affinity: {}
 

manifests/bucketeer/values.dev.yaml

@@ -173,6 +173,7 @@ batch-server:
  addr: localenv-redis-headless.default.svc.cluster.local:6379
  poolMaxIdle: 25
  poolMaxActive: 25
+ experimentLockTTL: 10m
 
  tls:
  service:

pkg/batch/cmd/server/server.go

@@ -73,6 +73,7 @@ type server struct {
  serviceTokenPath *string
  timezone *string
  refreshInterval *time.Duration
+ experimentLockTTL *time.Duration
  webURL *string
  oauthPublicKeyPath *string
  oauthAudience *string
@@ -213,6 +214,9 @@ func RegisterCommand(r cli.CommandRegistry, p cli.ParentCommand) cli.Command {
  "non-persistent-redis-pool-max-active",
  "Maximum number of connections allocated by the non-persistent redis connections pool at a given time.",
  ).Default("10").Int(),
+ experimentLockTTL: cmd.Flag("experiment-lock-ttl",
+ "The ttl for experiment calculator lock").
+ Default("10m").Duration(),
  }
  r.RegisterCommand(server)
  return server
@@ -456,7 +460,7 @@ func (s *server) Run(ctx context.Context, metrics metrics.Metrics, logger *zap.L
  experimentClient,
  eventCounterClient,
  mysqlClient,
- calculator.NewExperimentLock(nonPersistentRedisClient),
+ calculator.NewExperimentLock(nonPersistentRedisClient, *s.experimentLockTTL),
  location,
  jobs.WithTimeout(5*time.Minute),
  jobs.WithLogger(logger),

pkg/batch/jobs/calculator/experiment_lock.go

@@ -24,8 +24,7 @@ import (
 )
 
 const (
- experimentLockPrefix = "experiment-calculate:"
- experimentLockTTL = time.Minute
+ experimentLockKind = "experiment_lock"
 )
 
 // ExperimentLock represents a distributed lock for experiments
@@ -34,25 +33,25 @@ type ExperimentLock struct {
 }
 
 // NewExperimentLock creates a new ExperimentLock
-func NewExperimentLock(client redisv3.Client) *ExperimentLock {
+func NewExperimentLock(client redisv3.Client, lockTTL time.Duration) *ExperimentLock {
  return &ExperimentLock{
- lock: lock.NewDistributedLock(client, experimentLockTTL),
+ lock: lock.NewDistributedLock(client, lockTTL),
  }
 }
 
 // Lock attempts to acquire the lock for a specific experiment
 func (el *ExperimentLock) Lock(ctx context.Context, environmentID, experimentID string) (bool, string, error) {
- lockKey := el.getLockKey(environmentID, experimentID)
+ lockKey := el.newLockKey(environmentID, experimentID)
  return el.lock.Lock(ctx, lockKey)
 }
 
 // Unlock releases the lock for a specific experiment
 func (el *ExperimentLock) Unlock(ctx context.Context, environmentID, experimentID, value string) (bool, error) {
- lockKey := el.getLockKey(environmentID, experimentID)
+ lockKey := el.newLockKey(environmentID, experimentID)
  return el.lock.Unlock(ctx, lockKey, value)
 }
 
-// getLockKey generates the lock key for a specific experiment
-func (el *ExperimentLock) getLockKey(environmentID, experimentID string) string {
- return fmt.Sprintf("%s%s:%s", experimentLockPrefix, environmentID, experimentID)
+// newLockKey generates the lock key for a specific experiment
+func (el *ExperimentLock) newLockKey(environmentID, experimentID string) string {
+ return fmt.Sprintf("%s:%s:%s", environmentID, experimentLockKind, experimentID)
 }