improving retry dynamic for scale-up sqs because of the ephemeral run…

…ners (#1383)

Improve retry dynamic control for scale-up sqs

terraform-aws-github-runner/main.tf

@@ -22,15 +22,27 @@ resource "random_string" "random" {
   upper   = false
 }
 
+resource "aws_sqs_queue" "queued_builds_dead_letter" {
+  name                        = "${var.environment}-queued-builds-dead-letter.fifo"
+  fifo_queue                  = true
+  redrive_allow_policy        = jsonencode({
+    redrivePermission = "allowAll",
+  })
+  tags                        = var.tags
+}
+
 resource "aws_sqs_queue" "queued_builds" {
   name                        = "${var.environment}-queued-builds.fifo"
-  visibility_timeout_seconds  = var.runners_scale_up_lambda_timeout
+  visibility_timeout_seconds  = var.runners_scale_up_sqs_visibility_timeout
   fifo_queue                  = true
   content_based_deduplication = true
   max_message_size            = 1024
-  message_retention_seconds   = 5800
-
-  tags = var.tags
+  message_retention_seconds   = var.runners_scale_up_sqs_max_retry * var.runners_scale_up_sqs_visibility_timeout + 100
+  redrive_policy              = jsonencode({
+    deadLetterTargetArn = aws_sqs_queue.queued_builds_dead_letter.arn
+    maxReceiveCount     = var.runners_scale_up_sqs_max_retry
+  })
+  tags                        = var.tags
 }
 
 module "webhook" {

terraform-aws-github-runner/variables.tf

@@ -87,6 +87,18 @@ variable "runners_lambda_zip" {
   default     = null
 }
 
+variable "runners_scale_up_sqs_max_retry" {
+  description = "max retry count for messages in the scale up sqs."
+  type        = number
+  default     = 3
+}
+
+variable "runners_scale_up_sqs_visibility_timeout" {
+  description = "Time out for visibility of messages in the scale up sqs."
+  type        = number
+  default     = 180
+}
+
 variable "runners_scale_up_lambda_timeout" {
   description = "Time out for the scale up lambda in seconds."
   type        = number