Skip to content

Commit

Permalink
feat(scripts): add opensearch index mapping script (reanahub#827)
Browse files Browse the repository at this point in the history
  • Loading branch information
jlemesh committed Aug 30, 2024
1 parent 049691c commit 63c6220
Show file tree
Hide file tree
Showing 3 changed files with 242 additions and 0 deletions.
65 changes: 65 additions & 0 deletions scripts/opensearch/job_log_mapping.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
{
"dynamic": "strict",
"properties": {
"@timestamp": {
"type": "date"
},
"kubernetes": {
"properties": {
"container_image": {
"type": "text"
},
"container_name": {
"type": "text"
},
"docker_id": {
"type": "text"
},
"host": {
"type": "text"
},
"labels": {
"properties": {
"batch.kubernetes.io/controller-uid": { "type": "text" },
"batch.kubernetes.io/job-name": { "type": "text" },
"controller-uid": { "type": "text" },
"job-name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"reana-run-job-workflow-uuid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
}
}
},
"namespace_name": {
"type": "text"
},
"pod_id": {
"type": "text"
},
"pod_name": {
"type": "text"
}
}
},
"log": {
"type": "text"
},
"stream": {
"type": "text"
},
"time": {
"type": "date"
}
}
}
111 changes: 111 additions & 0 deletions scripts/opensearch/opensearch.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
#!/bin/bash
#
# This file is part of REANA.
# Copyright (C) 2024 CERN.
#
# REANA is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.

set -euo pipefail

# Usage examples:
#
# Local environment with OpenSearch security plugin off:
# ./opensearch.sh
#
# Local environment with OpenSearch security plugin on (kubectl should be installed):
# OPENSEARCH_CN=opensearch-cluster-master.default.svc.cluster.local OPENSEARCH_PROTOCOL=https KUBECTL_CERTS=true ./opensearch.sh
#
# Local environment to remote OpenSearch, with OpenSearch security plugin on (kubectl should be installed):
# OPENSEARCH_CN=opensearch-cluster-master.default.svc.cluster.local OPENSEARCH_HOST=opensearch.cern.ch OPENSEARCH_PROTOCOL=https KUBECTL_CERTS=true ./opensearch.sh
#
# From inside OpenSearch pod, with OpenSearch security plugin on:
# OPENSEARCH_CN=opensearch-cluster-master.default.svc.cluster.local OPENSEARCH_PROTOCOL=https CERTS_FROM_FILE=true ./opensearch.sh
#
# Local environment to remote OpenSearch, with OpenSearch security plugin on:
# OPENSEARCH_HOST=opensearch.cern.ch OPENSEARCH_PROTOCOL=https CERTS_FROM_FILE=true CERT_DIR=. ./opensearch.sh

# OpenSearch address configuration
OPENSEARCH_PROTOCOL="${OPENSEARCH_PROTOCOL:-http}" # http or https
OPENSEARCH_HOST="${OPENSEARCH_HOST:-localhost}"
OPENSEARCH_PORT="${OPENSEARCH_PORT:-9200}"
OPENSEARCH_CN="${OPENSEARCH_CN:-$OPENSEARCH_HOST}" # OpenSearch common name in node's TLS certificate, defaults to OPENSEARCH_HOST
OPENSEARCH_ADDRESS=$OPENSEARCH_PROTOCOL://$OPENSEARCH_CN:$OPENSEARCH_PORT

# Index mapping configuration
JOB_LOG_IDX="${JOB_LOG_IDX:-fluentbit-job_log}"
WORKFLOW_LOG_IDX="${WORKFLOW_LOG_IDX:-fluentbit-workflow_log}"
JOB_LOG_MAPPING_FILE_PATH="${JOB_LOG_MAPPING_FILE_PATH:-job_log_mapping.json}"
WORKFLOW_LOG_MAPPING_FILE_PATH="${WORKFLOW_LOG_MAPPING_FILE_PATH:-workflow_log_mapping.json}"

# TLS configuration
KUBECTL_CERTS="${KUBECTL_CERTS:-false}" # Use admin certificates from Kubernetes secret; makes cURL use TLS while connecting to OpenSearch
CERTS_FROM_FILE="${CERTS_FROM_FILE:-false}" # Use admin certificates from files; makes cURL use TLS while connecting to OpenSearch
CERT_DIR="${CERT_DIR:-config/certs}" # If CERTS_FROM_FILE is true, the directory where the certificates are stored (files should be named ca.crt, admin.crt and admin.key)
KUBECTL_SECRET_NAME="${KUBECTL_SECRET_NAME:-reana-opensearch-tls-secret}" # If KUBECTL_CERTS is true, the secret name to use

# Execute cURL when TLS is enabled
curl_exec_secure() {
curl --connect-to "$OPENSEARCH_CN:$OPENSEARCH_PORT:$OPENSEARCH_HOST" \
--cacert <(echo "$CACERT") --cert <(echo "$CERT") --key <(echo "$KEY") \
-H 'Content-Type: application/json' "$@"
echo ""
}

# Execute cURL when TLS is disabled
curl_exec() {
curl -H 'Content-Type: application/json' "$@"
echo ""
}

# Setup indices and mappings
run() {
echo "OpenSearch adderess: $OPENSEARCH_ADDRESS."
echo "Create indices (ignore errors if already created):"
echo -n "$JOB_LOG_IDX response: "
$1 -XPUT "$OPENSEARCH_ADDRESS/$JOB_LOG_IDX"

echo -n "$WORKFLOW_LOG_IDX response: "
$1 -XPUT "$OPENSEARCH_ADDRESS/$WORKFLOW_LOG_IDX"


echo "Create mappings:"
echo -n "$JOB_LOG_IDX response: "
$1 -XPUT "$OPENSEARCH_ADDRESS/$JOB_LOG_IDX/_mapping" \
--data "@$JOB_LOG_MAPPING_FILE_PATH"

echo -n "$WORKFLOW_LOG_IDX response: "
$1 -XPUT "$OPENSEARCH_ADDRESS/$WORKFLOW_LOG_IDX/_mapping" \
--data "@$WORKFLOW_LOG_MAPPING_FILE_PATH"


echo "Update mappings:"
echo -n "$JOB_LOG_IDX response: "
$1 -XPOST "$OPENSEARCH_ADDRESS/$JOB_LOG_IDX/_update_by_query"

echo -n "$WORKFLOW_LOG_IDX response: "
$1 -XPOST "$OPENSEARCH_ADDRESS/$WORKFLOW_LOG_IDX/_update_by_query"

echo "Done."
}

# Execute script
if [[ $KUBECTL_CERTS = true ]]; then
echo "Using certificates from Kubernetes secret."
CACERT=$(kubectl get secret "$KUBECTL_SECRET_NAME" \
-ogo-template='{{ index .data "ca.crt" | base64decode }}')
CERT=$(kubectl get secret "$KUBECTL_SECRET_NAME" \
-ogo-template='{{ index .data "admin.crt" | base64decode }}')
KEY=$(kubectl get secret "$KUBECTL_SECRET_NAME" \
-ogo-template='{{ index .data "admin.key" | base64decode }}')
run curl_exec_secure
elif [[ $CERTS_FROM_FILE = true ]]; then
echo "Using certificates from files."
CACERT=$(cat "$CERT_DIR"/ca.crt)
CERT=$(cat "$CERT_DIR"/admin.crt)
KEY=$(cat "$CERT_DIR"/admin.key)
run curl_exec_secure
else
echo "Not using TLS."
run curl_exec
fi
66 changes: 66 additions & 0 deletions scripts/opensearch/workflow_log_mapping.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
{
"dynamic": "strict",
"properties": {
"@timestamp": {
"type": "date"
},
"kubernetes": {
"properties": {
"container_image": {
"type": "text"
},
"container_name": {
"type": "text"
},
"docker_id": {
"type": "text"
},
"host": {
"type": "text"
},
"labels": {
"properties": {
"batch.kubernetes.io/controller-uid": { "type": "text" },
"batch.kubernetes.io/job-name": { "type": "text" },
"controller-uid": { "type": "text" },
"job-name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"reana-run-batch-workflow-uuid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"reana_workflow_mode": { "type": "text" }
}
},
"namespace_name": {
"type": "text"
},
"pod_id": {
"type": "text"
},
"pod_name": {
"type": "text"
}
}
},
"log": {
"type": "text"
},
"stream": {
"type": "text"
},
"time": {
"type": "date"
}
}
}

0 comments on commit 63c6220

Please sign in to comment.