From 4d11be372dcb8fa64cd99195018c1b2ab82393d5 Mon Sep 17 00:00:00 2001 From: Hector Machin Date: Wed, 13 Dec 2023 14:07:56 -0500 Subject: [PATCH 1/5] Adding a stac-server initialization period to wait on successful create_indices --- modules/stac-server/inputs.tf | 6 ++++++ modules/stac-server/opensearch_domain.tf | 2 +- modules/stac-server/opensearch_serverless.tf | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/stac-server/inputs.tf b/modules/stac-server/inputs.tf index 6f47407..d46a4fd 100644 --- a/modules/stac-server/inputs.tf +++ b/modules/stac-server/inputs.tf @@ -316,3 +316,9 @@ variable deploy_stac_server_opensearch_serverless { default = false description = "Deploy FilmDrop Stac-Server with OpenSearch Serverless. If False, Stac-server will be deployed with a classic OpenSearch domain." } + +variable "stac_server_index_initialization_period" { + description = "Settlement period to wait for initialization of the required indices in OpenSearch prior to start ingest." + type = number + default = 10 +} diff --git a/modules/stac-server/opensearch_domain.tf b/modules/stac-server/opensearch_domain.tf index 04b3e68..381a883 100644 --- a/modules/stac-server/opensearch_domain.tf +++ b/modules/stac-server/opensearch_domain.tf @@ -335,7 +335,7 @@ export AWS_REGION=${data.aws_region.current.name} echo "Creating indices on Stac Server OpenSearch Service." aws lambda invoke --function-name ${aws_lambda_function.stac_server_ingest.function_name} --cli-binary-format raw-in-base64-out --payload '{ "create_indices": true }' output - +sleep ${var.stac_server_index_initialization_period} EOF } diff --git a/modules/stac-server/opensearch_serverless.tf b/modules/stac-server/opensearch_serverless.tf index 1daa0f5..8de79d5 100644 --- a/modules/stac-server/opensearch_serverless.tf +++ b/modules/stac-server/opensearch_serverless.tf @@ -113,7 +113,7 @@ export AWS_REGION=${data.aws_region.current.name} echo "Creating indices on Stac Server OpenSearch Service." aws lambda invoke --function-name ${aws_lambda_function.stac_server_ingest.function_name} --cli-binary-format raw-in-base64-out --payload '{ "create_indices": true }' output - +sleep ${var.stac_server_index_initialization_period} EOF } From c56676666405b914be8079f1a125141a84c536aa Mon Sep 17 00:00:00 2001 From: Hector Machin Date: Wed, 13 Dec 2023 15:33:19 -0500 Subject: [PATCH 2/5] Removing stac-server null_resource lambda invocations and setting dependencies between ongoing and historical ingest --- modules/stac-server/inputs.tf | 6 --- modules/stac-server/opensearch_domain.tf | 47 ++++---------------- modules/stac-server/opensearch_serverless.tf | 24 +++------- profiles/stac-server/main.tf | 1 + 4 files changed, 14 insertions(+), 64 deletions(-) diff --git a/modules/stac-server/inputs.tf b/modules/stac-server/inputs.tf index d46a4fd..6f47407 100644 --- a/modules/stac-server/inputs.tf +++ b/modules/stac-server/inputs.tf @@ -316,9 +316,3 @@ variable deploy_stac_server_opensearch_serverless { default = false description = "Deploy FilmDrop Stac-Server with OpenSearch Serverless. If False, Stac-server will be deployed with a classic OpenSearch domain." } - -variable "stac_server_index_initialization_period" { - description = "Settlement period to wait for initialization of the required indices in OpenSearch prior to start ingest." - type = number - default = 10 -} diff --git a/modules/stac-server/opensearch_domain.tf b/modules/stac-server/opensearch_domain.tf index 381a883..6dc5761 100644 --- a/modules/stac-server/opensearch_domain.tf +++ b/modules/stac-server/opensearch_domain.tf @@ -287,27 +287,10 @@ resource "aws_lambda_function" "stac_server_opensearch_user_initializer" { ] } -resource "null_resource" "invoke_stac_server_opensearch_user_initializer" { - count = var.deploy_stac_server_opensearch_serverless ? 0 : 1 - triggers = { - INITIALIZER_LAMBDA = aws_lambda_function.stac_server_opensearch_user_initializer.function_name - OPENSEARCH_HOST = var.deploy_stac_server_opensearch_serverless ? aws_opensearchserverless_collection.stac_server_opensearch_serverless_collection[0].collection_endpoint : aws_opensearch_domain.stac_server_opensearch_domain[0].endpoint - OPENSEARCH_MASTER_CREDS_SECRET_ARN = aws_secretsmanager_secret.opensearch_master_password_secret.arn - OPENSEARCH_USER_CREDS_SECRET_ARN = aws_secretsmanager_secret.opensearch_stac_user_password_secret.arn - REGION = data.aws_region.current.name - } - - provisioner "local-exec" { - interpreter = ["bash", "-ec"] - command = < Date: Wed, 13 Dec 2023 15:36:26 -0500 Subject: [PATCH 3/5] Adding missing comma --- profiles/stac-server/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiles/stac-server/main.tf b/profiles/stac-server/main.tf index 965d82e..f01ce39 100644 --- a/profiles/stac-server/main.tf +++ b/profiles/stac-server/main.tf @@ -82,7 +82,7 @@ module "ongoing_ingest" { destination_collections_max_long = var.stac_server_inputs.ingest.destination_collections_max_long depends_on = [ - module.stac-server + module.stac-server, module.historical_ingest ] } From a6ddb023e239746e4dc630d2beabb0db061472c0 Mon Sep 17 00:00:00 2001 From: Hector Machin Date: Wed, 13 Dec 2023 16:17:47 -0500 Subject: [PATCH 4/5] Making the user initialization optional if running opensearch serverless --- modules/stac-server/opensearch_domain.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/stac-server/opensearch_domain.tf b/modules/stac-server/opensearch_domain.tf index 6dc5761..baa2534 100644 --- a/modules/stac-server/opensearch_domain.tf +++ b/modules/stac-server/opensearch_domain.tf @@ -288,6 +288,7 @@ resource "aws_lambda_function" "stac_server_opensearch_user_initializer" { } resource "aws_lambda_invocation" "invoke_stac_server_opensearch_user_initializer" { + count = var.deploy_stac_server_opensearch_serverless ? 0 : 1 function_name = aws_lambda_function.stac_server_opensearch_user_initializer.function_name input = "{}" From ca5886d457c3f27b735ee3f4ee69b582e97af427 Mon Sep 17 00:00:00 2001 From: Hector Machin Date: Wed, 13 Dec 2023 17:03:05 -0500 Subject: [PATCH 5/5] Adding a comment to better explain the depends_on between ongoing and historical ingest --- profiles/stac-server/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiles/stac-server/main.tf b/profiles/stac-server/main.tf index f01ce39..80ffb54 100644 --- a/profiles/stac-server/main.tf +++ b/profiles/stac-server/main.tf @@ -83,6 +83,6 @@ module "ongoing_ingest" { depends_on = [ module.stac-server, - module.historical_ingest + module.historical_ingest # `historical_ingest` creates the collection, and without this `depends_on` a race condition can occur where the subscription for ongoing ingest can run prior to initializing the collection, leading to poorly mapped index and a bad stac-server state ] }