Add cirrus lambdas (#73)

Element84 · Aug 20, 2024 · 1639ab0 · 1639ab0
1 parent 536ea84
commit 1639ab0
Show file tree

Hide file tree

Showing 37 changed files with 1,259 additions and 207 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -12,18 +12,26 @@ jobs:
     env:
       CI: true
       STAC_SERVER_TAG: v3.7.0
+      CIRRUS_TAG: v1.0.0a0
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
           node-version: "18"
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
       - uses: hashicorp/setup-terraform@v3
         with:
           terraform_version: "1.7.5"
 
       - name: Update stac-server lambdas
         id: update_stac_lambdas
-        run: ./scripts/update-lambdas.sh
+        run: ./scripts/update-stac-server-lambdas.bash
+
+      - name: Update cirrus lambda dist
+        id: update_cirrus_lambda_dist
+        run: ./scripts/update-cirrus-lambda-dist.bash
 
       - name: Terraform Init
         id: tf_init

diff --git a/.github/workflows/deploy-manual-tests.yml b/.github/workflows/deploy-manual-tests.yml
@@ -12,6 +12,7 @@ jobs:
     env:
       CI: true
       STAC_SERVER_TAG: v3.7.0
+      CIRRUS_TAG: v1.0.0a0
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -46,7 +47,11 @@ jobs:
 
       - name: Update stac-server lambdas
         id: update_stac_lambdas
-        run: ./scripts/update-lambdas.sh
+        run: ./scripts/update-stac-server-lambdas.bash
+
+      - name: Update cirrus lambda dist
+        id: update_cirrus_lambda_dist
+        run: ./scripts/update-cirrus-lambda-dist.bash
 
       - name: Configure Terraform Init Credentials
         id: init_creds

diff --git a/.github/workflows/release-tests.yml b/.github/workflows/release-tests.yml
@@ -14,6 +14,7 @@ jobs:
     env:
       CI: true
       STAC_SERVER_TAG: v3.7.0
+      CIRRUS_TAG: v1.0.0a0
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -48,7 +49,11 @@ jobs:
 
       - name: Update stac-server lambdas
         id: update_stac_lambdas
-        run: ./scripts/update-lambdas.sh
+        run: ./scripts/update-stac-server-lambdas.bash
+
+      - name: Update cirrus lambda dist
+        id: update_cirrus_lambda_dist
+        run: ./scripts/update-cirrus-lambda-dist.bash
 
       - name: Configure Terraform Init Credentials
         id: init_creds

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,15 +5,21 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Added
+
+- Builtin lambdas added to cirrus module along with script to update deployment zip
+
 ## 2.27.0 - 2024-05-31
 
-## Changed
+### Changed
 
 - Rolled back vpc infrastructure changes to support creation of VPC if `deploy_vpc` is set to `true`.
 
 ## 2.26.0 - 2024-05-29
 
-## Changed
+### Changed
 
 - Default to stac-server 3.8.0 and OpenSearch 2.13
 - For both `stac_server_inputs` and `titiler_inputs`, renamed
@@ -39,7 +45,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 - Added Cirrus terraform base resource set and new cirrus terraform module
 
-## Changed
+### Changed
 
 - `titiler_inputs.mosaic_titiler_release_tag` is now `titiler_inputs.version`
 - `cirrus_dashboard_inputs.cirrus_dashboard_release` is now `cirrus_dashboard_inputs.version`

diff --git a/ci.tfvars b/ci.tfvars
@@ -150,6 +150,7 @@ console_ui_inputs = {
 cirrus_inputs = {
   data_bucket    = "cirrus-data-bucket-name"
   payload_bucket = "cirrus-payload-bucket-name"
+  log_level      = "DEBUG"
   process = {
     sqs_timeout           = 180
     sqs_max_receive_count = 5
@@ -158,6 +159,27 @@ cirrus_inputs = {
     timestream_magnetic_store_retention_period_in_days = 93
     timestream_memory_store_retention_period_in_hours  = 24
   }
+  api_lambda = {
+    timeout = 10
+    memory  = 128
+  }
+  process_lambda = {
+    timeout              = 10
+    memory               = 128
+    reserved_concurrency = 16
+  }
+  update_state_lambda = {
+    timeout = 15
+    memory  = 128
+  }
+  pre_batch_lambda = {
+    timeout = 15
+    memory  = 128
+  }
+  post_batch_lambda = {
+    timeout = 15
+    memory  = 128
+  }
 }
 
 cirrus_dashboard_inputs = {

diff --git a/default.tfvars b/default.tfvars
@@ -154,6 +154,7 @@ console_ui_inputs = {
 cirrus_inputs = {
   data_bucket    = "cirrus-data-bucket-name"
   payload_bucket = "cirrus-payload-bucket-name"
+  log_level      = "DEBUG"
   process = {
     sqs_timeout           = 180
     sqs_max_receive_count = 5
@@ -162,6 +163,27 @@ cirrus_inputs = {
     timestream_magnetic_store_retention_period_in_days = 93
     timestream_memory_store_retention_period_in_hours  = 24
   }
+  api_lambda = {
+    timeout = 10
+    memory  = 128
+  }
+  process_lambda = {
+    timeout              = 10
+    memory               = 128
+    reserved_concurrency = 16
+  }
+  update_state_lambda = {
+    timeout = 15
+    memory  = 128
+  }
+  pre_batch_lambda = {
+    timeout = 15
+    memory  = 128
+  }
+  post_batch_lambda = {
+    timeout = 15
+    memory  = 128
+  }
 }
 
 cirrus_dashboard_inputs = {

diff --git a/inputs.tf b/inputs.tf
@@ -349,6 +349,7 @@ variable "cirrus_inputs" {
   type = object({
     data_bucket    = string
     payload_bucket = string
+    log_level      = string
     process = object({
       sqs_timeout           = number
       sqs_max_receive_count = number
@@ -357,10 +358,32 @@ variable "cirrus_inputs" {
       timestream_magnetic_store_retention_period_in_days = number
       timestream_memory_store_retention_period_in_hours  = number
     })
+    api_lambda = object({
+      timeout = number
+      memory  = number
+    })
+    process_lambda = object({
+      timeout              = number
+      memory               = number
+      reserved_concurrency = number
+    })
+    update_state_lambda = object({
+      timeout = number
+      memory  = number
+    })
+    pre_batch_lambda = object({
+      timeout = number
+      memory  = number
+    })
+    post_batch_lambda = object({
+      timeout = number
+      memory  = number
+    })
   })
   default = {
     data_bucket    = "cirrus-data-bucket-name"
     payload_bucket = "cirrus-payload-bucket-name"
+    log_level      = "INFO"
     process = {
       sqs_timeout           = 180
       sqs_max_receive_count = 5
@@ -369,6 +392,27 @@ variable "cirrus_inputs" {
       timestream_magnetic_store_retention_period_in_days = 93
       timestream_memory_store_retention_period_in_hours  = 24
     }
+    api_lambda = {
+      timeout = 10
+      memory  = 128
+    }
+    process_lambda = {
+      timeout              = 10
+      memory               = 128
+      reserved_concurrency = 16
+    }
+    update_state_lambda = {
+      timeout = 15
+      memory  = 128
+    }
+    pre_batch_lambda = {
+      timeout = 15
+      memory  = 128
+    }
+    post_batch_lambda = {
+      timeout = 15
+      memory  = 128
+    }
   }
 }
 

diff --git a/modules/cirrus/base/iam.tf b/modules/cirrus/base/iam.tf
@@ -1,84 +1,3 @@
-resource "aws_iam_role" "cirrus_batch_role" {
-  name_prefix = "${var.cirrus_prefix}-batch-role-"
-
-  assume_role_policy = <<EOF
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Action": "sts:AssumeRole",
-      "Principal": {
-        "Service": "ecs-tasks.amazonaws.com"
-      },
-      "Effect": "Allow",
-      "Condition": {
-        "StringEquals": {
-            "aws:SourceAccount": "${data.aws_caller_identity.current.account_id}"
-        },
-        "ArnLike": {
-            "aws:SourceArn": "arn:aws:ecs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*"
-        }
-      }
-    }
-  ]
-}
-EOF
-
-}
-
-resource "aws_iam_policy" "cirrus_batch_policy" {
-  name_prefix = "${var.cirrus_prefix}-batch-policy-"
-
-  policy = <<EOF
-{
-	"Version": "2012-10-17",
-	"Statement": [
-		{
-			"Action": [
-				"s3:PutObject"
-			],
-			"Resource": [
-				"arn:aws:s3:::${var.cirrus_data_bucket}*",
-				"arn:aws:s3:::${var.cirrus_payload_bucket}*"
-			],
-			"Effect": "Allow"
-		},
-		{
-			"Action": [
-				"s3:ListBucket",
-				"s3:GetObject",
-				"s3:GetBucketLocation"
-			],
-			"Resource": "*",
-			"Effect": "Allow"
-		},
-		{
-			"Action": "secretsmanager:GetSecretValue",
-			"Resource": [
-				"arn:aws:secretsmanager:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:secret:${var.cirrus_prefix}*"
-			],
-			"Effect": "Allow"
-		},
-		{
-			"Action": [
-				"lambda:GetFunction"
-			],
-			"Resource": [
-				"arn:aws:lambda:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:function:${var.cirrus_prefix}*"
-			],
-			"Effect": "Allow"
-		}
-	]
-}
-EOF
-
-}
-
-resource "aws_iam_role_policy_attachment" "cirrus_batch_role_policy_attachment" {
-  role       = aws_iam_role.cirrus_batch_role.name
-  policy_arn = aws_iam_policy.cirrus_batch_policy.arn
-}
-
 resource "aws_iam_role" "cirrus_instance_role" {
   name_prefix = "${var.cirrus_prefix}-instance-role-"
 
@@ -108,28 +27,3 @@ resource "aws_iam_instance_profile" "cirrus_instance_profile" {
   name_prefix = "${var.cirrus_prefix}-instance-profile-"
   role        = aws_iam_role.cirrus_instance_role.name
 }
-
-resource "aws_iam_role" "cirrus_ec2_spot_role" {
-  name_prefix = "${var.cirrus_prefix}-ec2-spot-role-"
-
-  assume_role_policy = <<EOF
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Action": "sts:AssumeRole",
-      "Principal": {
-        "Service": "spotfleet.amazonaws.com"
-      },
-      "Effect": "Allow"
-    }
-  ]
-}
-EOF
-
-}
-
-resource "aws_iam_role_policy_attachment" "cirrus_ec2_spot_role_policy_attachment" {
-  role       = aws_iam_role.cirrus_ec2_spot_role.name
-  policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEC2SpotFleetTaggingRole"
-}