Adding OpenSearch Serverless capability to stac-server module

.github/workflows/ci.yml

@@ -1,21 +1,34 @@
 name: Continuous integration
 
 on:
-  pull_request:
   push:
-    branches:
-      main
+    branches: ["main" ]
+  pull_request:
+    branches: ["main"]
 
 jobs:
   update-lambdas:
     runs-on: ubuntu-latest
     env:
       CI: true
-      STAC_SERVER_TAG: v2.2.3
+      STAC_SERVER_TAG: v3.1.0
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: '16'
+          node-version: '18'
+      - uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: "1.5.5"
+
       - name: Update stac-server lambdas
-        run: ./scripts/update-lambdas.sh
+        id: update_stac_lambdas
+        run: ./scripts/update-lambdas.sh
+
+      - name: Terraform Init
+        id: tf_init
+        run: terraform init
+
+      - name: Terraform Validate
+        id: tf_validate
+        run: terraform validate -no-color

.github/workflows/snyk-scan.yml

@@ -18,7 +18,6 @@ jobs:
     permissions:
       contents: read # for actions/checkout to fetch code
       security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
-      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
     runs-on: ubuntu-latest
     steps:
       - uses: actions/[email protected]

CHANGELOG.md

@@ -13,6 +13,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Fixed
 
+## 2.1.0
+
+### Added
+- Added OpenSearch Serverless capability to stac-server module
+
 ## 2.0.0
 
 ### Added

default.tfvars

@@ -24,7 +24,7 @@ sns_critical_subscriptions_map  = {}
 ##### APPLICATION VARIABLES ####
 stac_server_inputs  = {
   app_name                                      = "stac_server"
-  version                                       = "v2.2.3"
+  version                                       = "v3.1.0"
   domain_alias                                  = ""
   enable_transactions_extension                 = false
   collection_to_index_mappings                  = ""
@@ -96,6 +96,7 @@ deploy_vpc                          = false
 deploy_vpc_search                   = true
 deploy_log_archive                  = true
 deploy_alarms                       = false
+deploy_stac_opensearch_serverless   = true
 deploy_stac_server                  = true
 deploy_analytics                    = true
 deploy_titiler                      = true

filmdrop.tf

@@ -31,6 +31,7 @@ module "filmdrop" {
   deploy_log_archive                  = var.deploy_log_archive
   deploy_alarms                       = var.deploy_alarms
   deploy_stac_server                  = var.deploy_stac_server
+  deploy_stac_opensearch_serverless   = var.deploy_stac_opensearch_serverless
   deploy_analytics                    = var.deploy_analytics
   deploy_titiler                      = var.deploy_titiler
   deploy_console_ui                   = var.deploy_console_ui

flop

@@ -68,8 +68,8 @@ EOF
                             exit 1
                         fi
                         source $HOME/.nvm/nvm.sh
-                        nvm install v16
-                        nvm use v16
+                        nvm install v18
+                        nvm use v18
                         echo "Building stac-server..."
                         curl -L -f --no-progress-meter -o - "https://github.com/stac-utils/stac-server/archive/refs/tags/${STAC_SERVER_TAG}.tar.gz" | tar -xz
                         cd "$STAC_SERVER_DIR"
@@ -86,7 +86,7 @@ EOF
         destroy|rm)     export stac_opensearch_domain_name=`terraform output stac_opensearch_domain_name`
                         export stac_opensearch_domain_name="${stac_opensearch_domain_name//\"}"
                         export DELETE_OPENSEARCH_DOMAIN="no"
-                        if [[ "$stac_opensearch_domain_name" != "" && "$stac_opensearch_domain_name" != *"Warning"* ]]; then
+                        if [[ "$stac_opensearch_domain_name" != "" && !("$stac_opensearch_domain_name" =~ ".aoss.amazonaws.com") && "$stac_opensearch_domain_name" != *"Warning"* ]]; then
                             echo "We detected a Stac Server OpenSearch Domain $stac_opensearch_domain_name running in flop environment..."
                             echo "Do you really want to destroy the Stac Server OpenSearch domain along with other resources?"
                             echo "There is no undo. Only 'yes' will be accepted to confirm."
@@ -123,8 +123,8 @@ EOF
                             exit 1
                         fi
                         source $HOME/.nvm/nvm.sh
-                        nvm install v16
-                        nvm use v16
+                        nvm install v18
+                        nvm use v18
                         echo "Building stac-server..."
                         curl -L -f --no-progress-meter -o - "https://github.com/stac-utils/stac-server/archive/refs/tags/${STAC_SERVER_TAG}.tar.gz" | tar -xz
                         cd "$STAC_SERVER_DIR"

inputs.tf

@@ -108,7 +108,7 @@ variable stac_server_inputs {
   })
   default       = {
     app_name                                      = "stac_server"
-    version                                       = "v2.2.3"
+    version                                       = "v3.1.0"
     domain_alias                                  = ""
     enable_transactions_extension                 = false
     collection_to_index_mappings                  = ""
@@ -258,6 +258,12 @@ variable deploy_stac_server {
   description = "Deploy FilmDrop Stac-Server"
 }
 
+variable deploy_stac_opensearch_serverless {
+  type        = bool
+  default     = true
+  description = "Deploy FilmDrop Stac-Server with OpenSearch Serverless. If False, Stac-server will be deployed with a classic OpenSearch domain."
+}
+
 variable deploy_analytics {
   type        = bool
   default     = true

modules/stac-server/api.tf

@@ -5,7 +5,7 @@ resource "aws_lambda_function" "stac_server_api" {
   role             = aws_iam_role.stac_api_lambda_role.arn
   handler          = "index.handler"
   source_code_hash = filebase64sha256("${path.module}/lambda/api/api.zip")
-  runtime          = "nodejs16.x"
+  runtime          = "nodejs18.x"
   timeout          = var.api_lambda_timeout
   memory_size      = var.api_lambda_memory
 
@@ -20,7 +20,7 @@ resource "aws_lambda_function" "stac_server_api" {
       OPENSEARCH_HOST = (
         var.opensearch_host != ""
         ? var.opensearch_host
-        : aws_opensearch_domain.stac_server_opensearch_domain.endpoint
+        : local.opensearch_endpoint
       )
       ENABLE_TRANSACTIONS_EXTENSION = var.enable_transactions_extension
       STAC_API_ROOTPATH = (
@@ -34,14 +34,18 @@ resource "aws_lambda_function" "stac_server_api" {
         : var.stac_server_pre_hook_lambda_arn
       )
       POST_HOOK                        = var.stac_server_post_hook_lambda_arn
-      OPENSEARCH_CREDENTIALS_SECRET_ID = aws_secretsmanager_secret.opensearch_stac_user_password_secret.arn
+      OPENSEARCH_CREDENTIALS_SECRET_ID = var.deploy_stac_opensearch_serverless ? "" : aws_secretsmanager_secret.opensearch_stac_user_password_secret.arn
       COLLECTION_TO_INDEX_MAPPINGS     = var.collection_to_index_mappings
     }
   }
 
-  vpc_config {
-    subnet_ids         = var.vpc_subnet_ids
-    security_group_ids = var.vpc_security_group_ids
+  dynamic "vpc_config" {
+    for_each = { for i, j in [var.deploy_stac_opensearch_serverless] : i => j if var.deploy_stac_opensearch_serverless != true }
+
+    content {
+      subnet_ids         = var.vpc_subnet_ids
+      security_group_ids = var.vpc_security_group_ids
+    }
   }
 }
 

modules/stac-server/api_auth.tf

@@ -6,7 +6,7 @@ resource "aws_lambda_function" "stac_server_api_auth_pre_hook" {
   role             = aws_iam_role.stac_api_lambda_role.arn
   handler          = "index.handler"
   source_code_hash = filebase64sha256("${path.module}/lambda/pre-hook/pre-hook.zip")
-  runtime          = "nodejs16.x"
+  runtime          = "nodejs18.x"
   timeout          = var.pre_hook_lambda_timeout
   memory_size      = var.pre_hook_lambda_memory
 
@@ -16,9 +16,13 @@ resource "aws_lambda_function" "stac_server_api_auth_pre_hook" {
     }
   }
 
-  vpc_config {
-    subnet_ids         = var.vpc_subnet_ids
-    security_group_ids = var.vpc_security_group_ids
+  dynamic "vpc_config" {
+    for_each = { for i, j in [var.deploy_stac_opensearch_serverless] : i => j if var.deploy_stac_opensearch_serverless != true }
+
+    content {
+      subnet_ids         = var.vpc_subnet_ids
+      security_group_ids = var.vpc_security_group_ids
+    }
   }
 }
 

modules/stac-server/data.tf

@@ -18,3 +18,9 @@ resource "random_string" "user_init_lambda_zip_poke" {
   length  = 16
   special = false
 }
+
+locals {
+  name_prefix         = "fd-${var.project_name}-${var.stac_api_stage}"
+  opensearch_endpoint = var.deploy_stac_opensearch_serverless ? aws_opensearchserverless_collection.stac_server_opensearch_serverless_collection[0].collection_endpoint : aws_opensearch_domain.stac_server_opensearch_domain[0].endpoint
+  opensearch_domain   = var.deploy_stac_opensearch_serverless ? aws_opensearchserverless_collection.stac_server_opensearch_serverless_collection[0].dashboard_endpoint : aws_opensearch_domain.stac_server_opensearch_domain[0].domain_name
+}

modules/stac-server/iam.tf

@@ -62,6 +62,16 @@ locals {
         Resource = "*"
         Effect   = "Allow"
       },
+      {
+        Action   = ["kms:*"]
+        Resource = "*"
+        Effect   = "Allow"
+      },
+      {
+        Action   = ["aoss:*"]
+        Resource = "*"
+        Effect   = "Allow"
+      },
       {
         Action   = ["secretsmanager:*"]
         Resource = "*"

modules/stac-server/ingest.tf

@@ -5,23 +5,27 @@ resource "aws_lambda_function" "stac_server_ingest" {
   role                           = aws_iam_role.stac_api_lambda_role.arn
   handler                        = "index.handler"
   source_code_hash               = filebase64sha256("${path.module}/lambda/ingest/ingest.zip")
-  runtime                        = "nodejs16.x"
+  runtime                        = "nodejs18.x"
   timeout                        = var.ingest_lambda_timeout
   memory_size                    = var.ingest_lambda_memory
   reserved_concurrent_executions = var.reserved_concurrent_executions
 
   environment {
     variables = {
       LOG_LEVEL                        = var.log_level
-      OPENSEARCH_HOST                  = var.opensearch_host != "" ? var.opensearch_host : aws_opensearch_domain.stac_server_opensearch_domain.endpoint
-      OPENSEARCH_CREDENTIALS_SECRET_ID = aws_secretsmanager_secret.opensearch_stac_user_password_secret.arn
+      OPENSEARCH_HOST                  = var.opensearch_host != "" ? var.opensearch_host : local.opensearch_endpoint
+      OPENSEARCH_CREDENTIALS_SECRET_ID = var.deploy_stac_opensearch_serverless ? "" : aws_secretsmanager_secret.opensearch_stac_user_password_secret.arn
       COLLECTION_TO_INDEX_MAPPINGS     = var.collection_to_index_mappings
     }
   }
 
-  vpc_config {
-    subnet_ids         = var.vpc_subnet_ids
-    security_group_ids = var.vpc_security_group_ids
+  dynamic "vpc_config" {
+    for_each = { for i, j in [var.deploy_stac_opensearch_serverless] : i => j if var.deploy_stac_opensearch_serverless != true }
+
+    content {
+      subnet_ids         = var.vpc_subnet_ids
+      security_group_ids = var.vpc_security_group_ids
+    }
   }
 }
 
@@ -98,7 +102,7 @@ resource "aws_lambda_permission" "stac_server_ingest_sqs_lambda_permission" {
 resource "null_resource" "stac_server_ingest_create_indices" {
   triggers = {
     stac_server_ingest = aws_lambda_function.stac_server_ingest.function_name
-    opensearch_host    = var.opensearch_host != "" ? var.opensearch_host : aws_opensearch_domain.stac_server_opensearch_domain.endpoint
+    opensearch_host    = var.opensearch_host != "" ? var.opensearch_host : var.deploy_stac_opensearch_serverless ? aws_opensearchserverless_collection.stac_server_opensearch_serverless_collection[0].collection_endpoint : aws_opensearch_domain.stac_server_opensearch_domain[0].endpoint
   }
 
   provisioner "local-exec" {

modules/stac-server/inputs.tf

@@ -305,6 +305,8 @@ variable "opensearch_cluster_dedicated_master_count" {
   default     = 3
 }
 
-locals {
-  name_prefix = "fd-${var.project_name}-${var.stac_api_stage}"
-}
+variable deploy_stac_opensearch_serverless {
+  type        = bool
+  default     = true
+  description = "Deploy FilmDrop Stac-Server with OpenSearch Serverless. If False, Stac-server will be deployed with a classic OpenSearch domain."
+}

modules/stac-server/opensearch_domain.tf

@@ -3,8 +3,9 @@ resource "random_id" "suffix" {
 }
 
 resource "aws_opensearch_domain" "stac_server_opensearch_domain" {
-  domain_name    = lower(var.opensearch_stac_server_domain_name_override == null ? "${local.name_prefix}-stac-server" : var.opensearch_stac_server_domain_name_override)
-  engine_version = var.opensearch_version
+  count           = var.deploy_stac_opensearch_serverless ? 0 : 1
+  domain_name     = lower(var.opensearch_stac_server_domain_name_override == null ? "${local.name_prefix}-stac-server" : var.opensearch_stac_server_domain_name_override)
+  engine_version  = var.opensearch_version
 
   cluster_config {
     instance_type            = var.opensearch_cluster_instance_type
@@ -57,7 +58,7 @@ resource "aws_opensearch_domain" "stac_server_opensearch_domain" {
 
   vpc_options {
     subnet_ids         = var.vpc_subnet_ids
-    security_group_ids = [aws_security_group.opensearch_security_group.id]
+    security_group_ids = [aws_security_group.opensearch_security_group[0].id]
   }
 
   advanced_options = {
@@ -95,6 +96,7 @@ CONFIG
 }
 
 resource "aws_security_group" "opensearch_security_group" {
+  count       = var.deploy_stac_opensearch_serverless ? 0 : 1
   name        = "${local.name_prefix}-stac-server"
   description = "OpenSearch Security Group"
   vpc_id      = var.vpc_id
@@ -258,20 +260,23 @@ resource "aws_lambda_function" "stac_server_opensearch_user_initializer" {
 
   environment {
     variables = {
-      OPENSEARCH_HOST                    = var.opensearch_host != "" ? var.opensearch_host : aws_opensearch_domain.stac_server_opensearch_domain.endpoint
+      OPENSEARCH_HOST                    = var.opensearch_host != "" ? var.opensearch_host : local.opensearch_endpoint
       OPENSEARCH_MASTER_CREDS_SECRET_ARN = aws_secretsmanager_secret.opensearch_master_password_secret.arn
       OPENSEARCH_USER_CREDS_SECRET_ARN   = aws_secretsmanager_secret.opensearch_stac_user_password_secret.arn
       REGION                             = data.aws_region.current.name
     }
   }
 
-  vpc_config {
-    subnet_ids         = var.vpc_subnet_ids
-    security_group_ids = var.vpc_security_group_ids
+  dynamic "vpc_config" {
+    for_each = { for i, j in [var.deploy_stac_opensearch_serverless] : i => j if var.deploy_stac_opensearch_serverless != true }
+
+    content {
+      subnet_ids         = var.vpc_subnet_ids
+      security_group_ids = var.vpc_security_group_ids
+    }
   }
 
   depends_on = [
-    aws_opensearch_domain.stac_server_opensearch_domain,
     random_password.opensearch_master_password,
     aws_secretsmanager_secret.opensearch_master_password_secret,
     aws_secretsmanager_secret_version.opensearch_master_password_secret_version,
@@ -283,9 +288,10 @@ resource "aws_lambda_function" "stac_server_opensearch_user_initializer" {
 }
 
 resource "null_resource" "invoke_stac_server_opensearch_user_initializer" {
-  triggers = {
+  count     = var.deploy_stac_opensearch_serverless ? 0 : 1
+  triggers  = {
     INITIALIZER_LAMBDA                 = aws_lambda_function.stac_server_opensearch_user_initializer.function_name
-    OPENSEARCH_HOST                    = aws_opensearch_domain.stac_server_opensearch_domain.endpoint
+    OPENSEARCH_HOST                    = var.deploy_stac_opensearch_serverless ? aws_opensearchserverless_collection.stac_server_opensearch_serverless_collection[0].collection_endpoint : aws_opensearch_domain.stac_server_opensearch_domain[0].endpoint
     OPENSEARCH_MASTER_CREDS_SECRET_ARN = aws_secretsmanager_secret.opensearch_master_password_secret.arn
     OPENSEARCH_USER_CREDS_SECRET_ARN   = aws_secretsmanager_secret.opensearch_stac_user_password_secret.arn
     REGION                             = data.aws_region.current.name
@@ -304,7 +310,6 @@ EOF
   }
 
   depends_on = [
-    aws_opensearch_domain.stac_server_opensearch_domain,
     random_password.opensearch_master_password,
     aws_secretsmanager_secret.opensearch_master_password_secret,
     aws_secretsmanager_secret_version.opensearch_master_password_secret_version,