Adding OpenSearch Serverless capability to stac-server module (#7)

.github/workflows/ci.yml

@@ -1,21 +1,34 @@
 name: Continuous integration
 
 on:
- pull_request:
  push:
- branches:
- main
+ branches: ["main" ]
+ pull_request:
+ branches: ["main"]
 
 jobs:
  update-lambdas:
  runs-on: ubuntu-latest
  env:
  CI: true
- STAC_SERVER_TAG: v2.2.3
+ STAC_SERVER_TAG: v3.2.0
  steps:
  - uses: actions/checkout@v4
  - uses: actions/setup-node@v4
  with:
- node-version: '16'
+ node-version: '18'
+ - uses: hashicorp/setup-terraform@v3
+ with:
+ terraform_version: "1.5.5"
+
  - name: Update stac-server lambdas
- run: ./scripts/update-lambdas.sh
+ id: update_stac_lambdas
+ run: ./scripts/update-lambdas.sh
+
+ - name: Terraform Init
+ id: tf_init
+ run: terraform init
+
+ - name: Terraform Validate
+ id: tf_validate
+ run: terraform validate -no-color

.github/workflows/snyk-scan.yml

@@ -18,7 +18,6 @@ jobs:
  permissions:
  contents: read # for actions/checkout to fetch code
  security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
- actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
  runs-on: ubuntu-latest
  steps:
  - uses: actions/[email protected]

CHANGELOG.md

@@ -13,6 +13,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Fixed
 
+## 2.1.0
+
+### Added
+- Added OpenSearch Serverless capability to stac-server module
+
 ## 2.0.0
 
 ### Added

default.tfvars

@@ -24,7 +24,7 @@ sns_critical_subscriptions_map = {}
 ##### APPLICATION VARIABLES ####
 stac_server_inputs = {
  app_name = "stac_server"
- version = "v2.2.3"
+ version = "v3.2.0"
  domain_alias = ""
  enable_transactions_extension = false
  collection_to_index_mappings = ""
@@ -92,17 +92,18 @@ cirrus_dashboard_inputs = {
 
 ##### INFRASTRUCTURE FLAGS ####
 # To disable each flag: set to 'false'; to enable: set to 'true'
-deploy_vpc = false
-deploy_vpc_search = true
-deploy_log_archive = true
-deploy_alarms = false
-deploy_stac_server = true
-deploy_analytics = true
-deploy_titiler = true
-deploy_console_ui = true
-deploy_cirrus_dashboard = true
-deploy_local_stac_server_artifacts = false
-deploy_sample_data_bucket = false
+deploy_vpc = false
+deploy_vpc_search = true
+deploy_log_archive = true
+deploy_alarms = false
+deploy_stac_server_opensearch_serverless = false
+deploy_stac_server = true
+deploy_analytics = true
+deploy_titiler = true
+deploy_console_ui = true
+deploy_cirrus_dashboard = true
+deploy_local_stac_server_artifacts = false
+deploy_sample_data_bucket = false
 
 
 ##### STAC SAMPLE DATA ####

filmdrop.tf

@@ -6,36 +6,37 @@ module "filmdrop" {
  aws.main = aws.main
  }
 
- environment = var.environment
- project_name = var.project_name
- vpc_id = var.vpc_id
- vpc_cidr = var.vpc_cidr
- public_subnets_cidr_map = var.public_subnets_cidr_map
- private_subnets_cidr_map = var.private_subnets_cidr_map
- security_group_id = var.security_group_id
- sns_topics_map = var.sns_topics_map
- cloudwatch_warning_alarms_map = var.cloudwatch_warning_alarms_map
- cloudwatch_critical_alarms_map = var.cloudwatch_critical_alarms_map
- sns_warning_subscriptions_map = var.sns_warning_subscriptions_map
- sns_critical_subscriptions_map = var.sns_critical_subscriptions_map
- s3_access_log_bucket = var.s3_access_log_bucket
- s3_logs_archive_bucket = var.s3_logs_archive_bucket
- domain_zone = var.domain_zone
- stac_server_inputs = var.stac_server_inputs
- titiler_inputs = var.titiler_inputs
- analytics_inputs = var.analytics_inputs
- console_ui_inputs = var.console_ui_inputs
- cirrus_dashboard_inputs = var.cirrus_dashboard_inputs
- deploy_vpc = var.deploy_vpc
- deploy_vpc_search = var.deploy_vpc_search
- deploy_log_archive = var.deploy_log_archive
- deploy_alarms = var.deploy_alarms
- deploy_stac_server = var.deploy_stac_server
- deploy_analytics = var.deploy_analytics
- deploy_titiler = var.deploy_titiler
- deploy_console_ui = var.deploy_console_ui
- deploy_cirrus_dashboard = var.deploy_cirrus_dashboard
- deploy_local_stac_server_artifacts = var.deploy_local_stac_server_artifacts
- deploy_sample_data_bucket = var.deploy_sample_data_bucket
- project_sample_data_bucket_name = var.project_sample_data_bucket_name
+ environment = var.environment
+ project_name = var.project_name
+ vpc_id = var.vpc_id
+ vpc_cidr = var.vpc_cidr
+ public_subnets_cidr_map = var.public_subnets_cidr_map
+ private_subnets_cidr_map = var.private_subnets_cidr_map
+ security_group_id = var.security_group_id
+ sns_topics_map = var.sns_topics_map
+ cloudwatch_warning_alarms_map = var.cloudwatch_warning_alarms_map
+ cloudwatch_critical_alarms_map = var.cloudwatch_critical_alarms_map
+ sns_warning_subscriptions_map = var.sns_warning_subscriptions_map
+ sns_critical_subscriptions_map = var.sns_critical_subscriptions_map
+ s3_access_log_bucket = var.s3_access_log_bucket
+ s3_logs_archive_bucket = var.s3_logs_archive_bucket
+ domain_zone = var.domain_zone
+ stac_server_inputs = var.stac_server_inputs
+ titiler_inputs = var.titiler_inputs
+ analytics_inputs = var.analytics_inputs
+ console_ui_inputs = var.console_ui_inputs
+ cirrus_dashboard_inputs = var.cirrus_dashboard_inputs
+ deploy_vpc = var.deploy_vpc
+ deploy_vpc_search = var.deploy_vpc_search
+ deploy_log_archive = var.deploy_log_archive
+ deploy_alarms = var.deploy_alarms
+ deploy_stac_server = var.deploy_stac_server
+ deploy_stac_server_opensearch_serverless = var.deploy_stac_server_opensearch_serverless
+ deploy_analytics = var.deploy_analytics
+ deploy_titiler = var.deploy_titiler
+ deploy_console_ui = var.deploy_console_ui
+ deploy_cirrus_dashboard = var.deploy_cirrus_dashboard
+ deploy_local_stac_server_artifacts = var.deploy_local_stac_server_artifacts
+ deploy_sample_data_bucket = var.deploy_sample_data_bucket
+ project_sample_data_bucket_name = var.project_sample_data_bucket_name
 }

flop

@@ -68,8 +68,8 @@ EOF
  exit 1
  fi
  source $HOME/.nvm/nvm.sh
- nvm install v16
- nvm use v16
+ nvm install v18
+ nvm use v18
  echo "Building stac-server..."
  curl -L -f --no-progress-meter -o - "https://github.com/stac-utils/stac-server/archive/refs/tags/${STAC_SERVER_TAG}.tar.gz" | tar -xz
  cd "$STAC_SERVER_DIR"
@@ -86,7 +86,7 @@ EOF
  destroy|rm) export stac_opensearch_domain_name=`terraform output stac_opensearch_domain_name`
  export stac_opensearch_domain_name="${stac_opensearch_domain_name//\"}"
  export DELETE_OPENSEARCH_DOMAIN="no"
- if [[ "$stac_opensearch_domain_name" != "" && "$stac_opensearch_domain_name" != *"Warning"* ]]; then
+ if [[ "$stac_opensearch_domain_name" != "" && !("$stac_opensearch_domain_name" =~ ".aoss.amazonaws.com") && "$stac_opensearch_domain_name" != *"Warning"* ]]; then
  echo "We detected a Stac Server OpenSearch Domain $stac_opensearch_domain_name running in flop environment..."
  echo "Do you really want to destroy the Stac Server OpenSearch domain along with other resources?"
  echo "There is no undo. Only 'yes' will be accepted to confirm."
@@ -123,8 +123,8 @@ EOF
  exit 1
  fi
  source $HOME/.nvm/nvm.sh
- nvm install v16
- nvm use v16
+ nvm install v18
+ nvm use v18
  echo "Building stac-server..."
  curl -L -f --no-progress-meter -o - "https://github.com/stac-utils/stac-server/archive/refs/tags/${STAC_SERVER_TAG}.tar.gz" | tar -xz
  cd "$STAC_SERVER_DIR"

inputs.tf

@@ -108,7 +108,7 @@ variable stac_server_inputs {
  })
  default = {
  app_name = "stac_server"
- version = "v2.2.3"
+ version = "v3.2.0"
  domain_alias = ""
  enable_transactions_extension = false
  collection_to_index_mappings = ""
@@ -258,6 +258,12 @@ variable deploy_stac_server {
  description = "Deploy FilmDrop Stac-Server"
 }
 
+variable deploy_stac_server_opensearch_serverless {
+ type = bool
+ default = false
+ description = "Deploy FilmDrop Stac-Server with OpenSearch Serverless. If False, Stac-server will be deployed with a classic OpenSearch domain."
+}
+
 variable deploy_analytics {
  type = bool
  default = true

modules/stac-server/api.tf

@@ -5,7 +5,7 @@ resource "aws_lambda_function" "stac_server_api" {
  role = aws_iam_role.stac_api_lambda_role.arn
  handler = "index.handler"
  source_code_hash = filebase64sha256("${path.module}/lambda/api/api.zip")
- runtime = "nodejs16.x"
+ runtime = "nodejs18.x"
  timeout = var.api_lambda_timeout
  memory_size = var.api_lambda_memory
 
@@ -20,7 +20,7 @@ resource "aws_lambda_function" "stac_server_api" {
  OPENSEARCH_HOST = (
  var.opensearch_host != ""
  ? var.opensearch_host
- : aws_opensearch_domain.stac_server_opensearch_domain.endpoint
+ : local.opensearch_endpoint
  )
  ENABLE_TRANSACTIONS_EXTENSION = var.enable_transactions_extension
  STAC_API_ROOTPATH = (
@@ -34,14 +34,18 @@ resource "aws_lambda_function" "stac_server_api" {
  : var.stac_server_pre_hook_lambda_arn
  )
  POST_HOOK = var.stac_server_post_hook_lambda_arn
- OPENSEARCH_CREDENTIALS_SECRET_ID = aws_secretsmanager_secret.opensearch_stac_user_password_secret.arn
+ OPENSEARCH_CREDENTIALS_SECRET_ID = var.deploy_stac_server_opensearch_serverless ? "" : aws_secretsmanager_secret.opensearch_stac_user_password_secret.arn
  COLLECTION_TO_INDEX_MAPPINGS = var.collection_to_index_mappings
  }
  }
 
- vpc_config {
- subnet_ids = var.vpc_subnet_ids
- security_group_ids = var.vpc_security_group_ids
+ dynamic "vpc_config" {
+ for_each = { for i, j in [var.deploy_stac_server_opensearch_serverless] : i => j if var.deploy_stac_server_opensearch_serverless != true }
+
+ content {
+ subnet_ids = var.vpc_subnet_ids
+ security_group_ids = var.vpc_security_group_ids
+ }
  }
 }
 

modules/stac-server/api_auth.tf

@@ -6,7 +6,7 @@ resource "aws_lambda_function" "stac_server_api_auth_pre_hook" {
  role = aws_iam_role.stac_api_lambda_role.arn
  handler = "index.handler"
  source_code_hash = filebase64sha256("${path.module}/lambda/pre-hook/pre-hook.zip")
- runtime = "nodejs16.x"
+ runtime = "nodejs18.x"
  timeout = var.pre_hook_lambda_timeout
  memory_size = var.pre_hook_lambda_memory
 
@@ -16,9 +16,13 @@ resource "aws_lambda_function" "stac_server_api_auth_pre_hook" {
  }
  }
 
- vpc_config {
- subnet_ids = var.vpc_subnet_ids
- security_group_ids = var.vpc_security_group_ids
+ dynamic "vpc_config" {
+ for_each = { for i, j in [var.deploy_stac_server_opensearch_serverless] : i => j if var.deploy_stac_server_opensearch_serverless != true }
+
+ content {
+ subnet_ids = var.vpc_subnet_ids
+ security_group_ids = var.vpc_security_group_ids
+ }
  }
 }
 

modules/stac-server/data.tf

@@ -18,3 +18,9 @@ resource "random_string" "user_init_lambda_zip_poke" {
  length = 16
  special = false
 }
+
+locals {
+ name_prefix = "fd-${var.project_name}-${var.stac_api_stage}"
+ opensearch_endpoint = var.deploy_stac_server_opensearch_serverless ? aws_opensearchserverless_collection.stac_server_opensearch_serverless_collection[0].collection_endpoint : aws_opensearch_domain.stac_server_opensearch_domain[0].endpoint
+ opensearch_domain = var.deploy_stac_server_opensearch_serverless ? aws_opensearchserverless_collection.stac_server_opensearch_serverless_collection[0].dashboard_endpoint : aws_opensearch_domain.stac_server_opensearch_domain[0].domain_name
+}

modules/stac-server/iam.tf

@@ -62,6 +62,16 @@ locals {
  Resource = "*"
  Effect = "Allow"
  },
+ {
+ Action = ["kms:*"]
+ Resource = "*"
+ Effect = "Allow"
+ },
+ {
+ Action = ["aoss:*"]
+ Resource = "*"
+ Effect = "Allow"
+ },
  {
  Action = ["secretsmanager:*"]
  Resource = "*"

modules/stac-server/ingest.tf

@@ -5,23 +5,27 @@ resource "aws_lambda_function" "stac_server_ingest" {
  role = aws_iam_role.stac_api_lambda_role.arn
  handler = "index.handler"
  source_code_hash = filebase64sha256("${path.module}/lambda/ingest/ingest.zip")
- runtime = "nodejs16.x"
+ runtime = "nodejs18.x"
  timeout = var.ingest_lambda_timeout
  memory_size = var.ingest_lambda_memory
  reserved_concurrent_executions = var.reserved_concurrent_executions
 
  environment {
  variables = {
  LOG_LEVEL = var.log_level
- OPENSEARCH_HOST = var.opensearch_host != "" ? var.opensearch_host : aws_opensearch_domain.stac_server_opensearch_domain.endpoint
- OPENSEARCH_CREDENTIALS_SECRET_ID = aws_secretsmanager_secret.opensearch_stac_user_password_secret.arn
+ OPENSEARCH_HOST = var.opensearch_host != "" ? var.opensearch_host : local.opensearch_endpoint
+ OPENSEARCH_CREDENTIALS_SECRET_ID = var.deploy_stac_server_opensearch_serverless ? "" : aws_secretsmanager_secret.opensearch_stac_user_password_secret.arn
  COLLECTION_TO_INDEX_MAPPINGS = var.collection_to_index_mappings
  }
  }
 
- vpc_config {
- subnet_ids = var.vpc_subnet_ids
- security_group_ids = var.vpc_security_group_ids
+ dynamic "vpc_config" {
+ for_each = { for i, j in [var.deploy_stac_server_opensearch_serverless] : i => j if var.deploy_stac_server_opensearch_serverless != true }
+
+ content {
+ subnet_ids = var.vpc_subnet_ids
+ security_group_ids = var.vpc_security_group_ids
+ }
  }
 }
 
@@ -98,7 +102,7 @@ resource "aws_lambda_permission" "stac_server_ingest_sqs_lambda_permission" {
 resource "null_resource" "stac_server_ingest_create_indices" {
  triggers = {
  stac_server_ingest = aws_lambda_function.stac_server_ingest.function_name
- opensearch_host = var.opensearch_host != "" ? var.opensearch_host : aws_opensearch_domain.stac_server_opensearch_domain.endpoint
+ opensearch_host = var.opensearch_host != "" ? var.opensearch_host : var.deploy_stac_server_opensearch_serverless ? aws_opensearchserverless_collection.stac_server_opensearch_serverless_collection[0].collection_endpoint : aws_opensearch_domain.stac_server_opensearch_domain[0].endpoint
  }
 
  provisioner "local-exec" {

modules/stac-server/inputs.tf

@@ -305,6 +305,8 @@ variable "opensearch_cluster_dedicated_master_count" {
  default = 3
 }
 
-locals {
- name_prefix = "fd-${var.project_name}-${var.stac_api_stage}"
-}
+variable deploy_stac_server_opensearch_serverless {
+ type = bool
+ default = false
+ description = "Deploy FilmDrop Stac-Server with OpenSearch Serverless. If False, Stac-server will be deployed with a classic OpenSearch domain."
+}