From 26df12d4ca6b35d66a75076a4adb9330b4c7e4bc Mon Sep 17 00:00:00 2001 From: Marat Abrarov Date: Tue, 20 Jun 2023 04:22:07 +0300 Subject: [PATCH] Configuration for testing https://github.com/fluent/fluent-bit/pull/1560. --- .gitattributes | 6 + .gitignore | 2 + README.md | 1 + elastic-cluster/README.md | 7 + elastic-cluster/docker-compose.yaml | 158 ++++++++++++++++++ elastic-cluster/init.sh | 82 +++++++++ fluent-bit-es-cluster/README.md | 7 + fluent-bit-es-cluster/fluent-bit.conf | 17 ++ fluent-bit-es-cluster/upstream.conf | 17 ++ fluent-bit-es/README.md | 1 + fluent-bit-es/fluent-bit.conf | 18 ++ single-node-elastic-cluster/README.md | 6 + .../docker-compose.yaml | 54 ++++++ single-node-elastic-cluster/init.sh | 82 +++++++++ 14 files changed, 458 insertions(+) create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 README.md create mode 100644 elastic-cluster/README.md create mode 100644 elastic-cluster/docker-compose.yaml create mode 100644 elastic-cluster/init.sh create mode 100644 fluent-bit-es-cluster/README.md create mode 100644 fluent-bit-es-cluster/fluent-bit.conf create mode 100644 fluent-bit-es-cluster/upstream.conf create mode 100644 fluent-bit-es/README.md create mode 100644 fluent-bit-es/fluent-bit.conf create mode 100644 single-node-elastic-cluster/README.md create mode 100644 single-node-elastic-cluster/docker-compose.yaml create mode 100644 single-node-elastic-cluster/init.sh diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6ba8d72 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +* text=auto +*.sh text eol=lf +*.conf text eol=lf +*.bat text eol=crlf +*.cmd text eol=crlf +*.ps1 text eol=crlf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c086591 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/.idea/ +/build/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..250a2dc --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# Configurations for Elastic stack and related technologies diff --git a/elastic-cluster/README.md b/elastic-cluster/README.md new file mode 100644 index 0000000..55df4c2 --- /dev/null +++ b/elastic-cluster/README.md @@ -0,0 +1,7 @@ +# Docker Compose project for Elastic cluster + +Docker Compose project for Elastic cluster consisting of: + +1. 3 master / data / ingesting Elasticsearch nodes. +1. 1 coordinating Elasticsearch node. +1. 1 Kibana node diff --git a/elastic-cluster/docker-compose.yaml b/elastic-cluster/docker-compose.yaml new file mode 100644 index 0000000..a0e9fa4 --- /dev/null +++ b/elastic-cluster/docker-compose.yaml @@ -0,0 +1,158 @@ +version: '2.1' + +x-logging: &default-logging + driver: 'json-file' + options: + max-size: '12m' + max-file: '5' + +volumes: + elasticsearch1-data: + elasticsearch2-data: + elasticsearch3-data: + elasticsearch4-data: + kibana-data: + +services: + elasticsearch1: + image: 'elasticsearch:7.17.9' + mem_limit: '1024m' + mem_reservation: '512m' + ulimits: + memlock: + soft: -1 + hard: -1 + environment: + node.name: 'elasticsearch1' + cluster.name: 'elasticsearch' + discovery.seed_hosts: 'elasticsearch1,elasticsearch2,elasticsearch3' + cluster.initial_master_nodes: 'elasticsearch1,elasticsearch2,elasticsearch3' + node.master: 'true' + node.data: 'true' + node.attr.data: 'hot' + node.ingest: 'true' + node.ml: 'false' + node.transform: 'false' + node.remote_cluster_client: 'false' + xpack.ml.enabled: 'false' + xpack.transform.enabled: 'false' + xpack.security.enabled: 'true' + ELASTIC_PASSWORD: 'elastic' + ports: + - '9201:9200' + volumes: + - 'elasticsearch1-data:/usr/share/elasticsearch/data' + logging: *default-logging + + elasticsearch2: + image: 'elasticsearch:7.17.9' + mem_limit: '1024m' + mem_reservation: '512m' + ulimits: + memlock: + soft: -1 + hard: -1 + environment: + node.name: 'elasticsearch2' + cluster.name: 'elasticsearch' + discovery.seed_hosts: 'elasticsearch1,elasticsearch2,elasticsearch3' + cluster.initial_master_nodes: 'elasticsearch1,elasticsearch2,elasticsearch3' + bootstrap.memory_lock: 'true' + node.master: 'true' + node.data: 'true' + node.attr.data: 'hot' + node.ingest: 'true' + node.ml: 'false' + node.transform: 'false' + node.remote_cluster_client: 'false' + xpack.ml.enabled: 'false' + xpack.transform.enabled: 'false' + xpack.security.enabled: 'true' + ELASTIC_PASSWORD: 'elastic' + ports: + - '9202:9200' + volumes: + - 'elasticsearch2-data:/usr/share/elasticsearch/data' + logging: *default-logging + + elasticsearch3: + image: 'elasticsearch:7.17.9' + mem_limit: '1024m' + mem_reservation: '512m' + ulimits: + memlock: + soft: -1 + hard: -1 + environment: + node.name: 'elasticsearch3' + cluster.name: 'elasticsearch' + discovery.seed_hosts: 'elasticsearch1,elasticsearch2,elasticsearch3' + cluster.initial_master_nodes: 'elasticsearch1,elasticsearch2,elasticsearch3' + bootstrap.memory_lock: 'true' + node.master: 'true' + node.data: 'true' + node.attr.data: 'hot' + node.ingest: 'true' + node.ml: 'false' + node.transform: 'false' + node.remote_cluster_client: 'false' + xpack.ml.enabled: 'false' + xpack.transform.enabled: 'false' + xpack.security.enabled: 'true' + ELASTIC_PASSWORD: 'elastic' + ports: + - '9203:9200' + volumes: + - 'elasticsearch3-data:/usr/share/elasticsearch/data' + logging: *default-logging + + elasticsearch4: + image: 'elasticsearch:7.17.9' + mem_limit: '1024m' + mem_reservation: '512m' + ulimits: + memlock: + soft: -1 + hard: -1 + environment: + node.name: 'elasticsearch4' + cluster.name: 'elasticsearch' + discovery.seed_hosts: 'elasticsearch1,elasticsearch2,elasticsearch3' + cluster.initial_master_nodes: 'elasticsearch1,elasticsearch2,elasticsearch3' + bootstrap.memory_lock: 'true' + node.master: 'false' + node.voting_only: 'false' + node.data: 'false' + node.ingest: 'false' + node.ml: 'false' + node.transform: 'false' + node.remote_cluster_client: 'false' + xpack.ml.enabled: 'false' + xpack.transform.enabled: 'false' + xpack.security.enabled: 'true' + ELASTIC_PASSWORD: 'elastic' + volumes: + - 'elasticsearch4-data:/usr/share/elasticsearch/data' + logging: *default-logging + + kibana: + image: 'kibana:7.17.9' + mem_limit: '1024m' + mem_reservation: '512m' + ulimits: + memlock: + soft: -1 + hard: -1 + environment: + SERVER_PUBLICBASEURL: 'http://localhost:5601' + ELASTICSEARCH_HOSTS: 'http://elasticsearch4:9200' + xpack.security.enabled: 'true' + ELASTICSEARCH_USERNAME: 'elastic' + ELASTICSEARCH_PASSWORD: 'elastic' + TELEMETRY_ENABLED: 'false' + NEWSFEED_ENABLED: 'false' + ports: + - '5601:5601' + volumes: + - 'kibana-data:/usr/share/kibana/data' + logging: *default-logging diff --git a/elastic-cluster/init.sh b/elastic-cluster/init.sh new file mode 100644 index 0000000..e5f8fbc --- /dev/null +++ b/elastic-cluster/init.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +set -e + +DOCKER_HOST='localhost' + +elasticsearch_api_user='elastic' +elasticsearch_api_password='elastic' +elasticsearch_api_url_base="http://${DOCKER_HOST}:9201" +kibana_api_url_base="http://${DOCKER_HOST}:5601" +elasticsearch_index_alias="fluent-bit" +elasticsearch_index_name="${elasticsearch_index_alias}-000001" +kibana_index_pattern_id="${elasticsearch_index_alias}" +kibana_index_pattern_title="${elasticsearch_index_alias}-*" + +response_status_code="$(curl -ks -X POST \ + -o /dev/null -w '%{http_code}\n' \ + --user "${elasticsearch_api_user}:${elasticsearch_api_password}" \ + "${elasticsearch_api_url_base}/_license/start_trial?acknowledge=true")" +if [[ "${response_status_code}" -ne 200 ]]; then + echo "Failed to set Elasticsearch trial license" >&2 + exit 1 +fi + +response_status_code="$(curl -s -X PUT \ + -o /dev/null -w '%{http_code}\n' \ + --user "${elasticsearch_api_user}:${elasticsearch_api_password}" \ + -H 'Content-Type: application/json' \ + -d \ +"{ + \"aliases\": { + \"${elasticsearch_index_alias}\": {} + }, + \"mappings\": { + \"properties\": { + \"@timestamp\": { + \"type\": \"date\" + }, + \"message\": { + \"type\": \"text\", + \"fields\": { + \"keyword\": { + \"type\": \"text\" + } + } + } + } + }, + \"settings\": { + \"index\": { + \"number_of_shards\": 2, + \"number_of_replicas\": 1, + \"refresh_interval\": \"30s\" + } + } +} +" \ + "${elasticsearch_api_url_base}/${elasticsearch_index_name}")" +if [[ "${response_status_code}" -ne 200 ]]; then + echo "Failed to create ${elasticsearch_index_name} Elasticsearch index" >&2 + exit 1 +fi + +response_status_code="$(curl -s -X POST \ + -o /dev/null -w '%{http_code}\n' \ + --user "${elasticsearch_api_user}:${elasticsearch_api_password}" \ + -H 'kbn-xsrf: required_header' \ + -H 'Content-Type: application/json' \ + -d \ +"{ + \"attributes\": { + \"title\": \"${kibana_index_pattern_title}\", + \"timeFieldName\": \"@timestamp\", + \"fields\": \"[ { \\\"name\\\": \\\"@timestamp\\\", \\\"type\\\": \\\"date\\\", \\\"esTypes\\\": [ \\\"date\\\" ], \\\"searchable\\\": true, \\\"aggregatable\\\": true, \\\"readFromDocValues\\\": true, \\\"metadata_field\\\": false }, { \\\"name\\\": \\\"_id\\\", \\\"type\\\": \\\"string\\\", \\\"esTypes\\\": [ \\\"_id\\\" ], \\\"searchable\\\": true, \\\"aggregatable\\\": true, \\\"readFromDocValues\\\": false, \\\"metadata_field\\\": true }, { \\\"name\\\": \\\"_index\\\", \\\"type\\\": \\\"string\\\", \\\"esTypes\\\": [ \\\"_index\\\" ], \\\"searchable\\\": true, \\\"aggregatable\\\": true, \\\"readFromDocValues\\\": false, \\\"metadata_field\\\": true }, { \\\"name\\\": \\\"_score\\\", \\\"type\\\": \\\"number\\\", \\\"searchable\\\": false, \\\"aggregatable\\\": false, \\\"readFromDocValues\\\": false, \\\"metadata_field\\\": true }, { \\\"name\\\": \\\"_source\\\", \\\"type\\\": \\\"_source\\\", \\\"esTypes\\\": [ \\\"_source\\\" ], \\\"searchable\\\": false, \\\"aggregatable\\\": false, \\\"readFromDocValues\\\": false, \\\"metadata_field\\\": true }, { \\\"name\\\": \\\"_type\\\", \\\"type\\\": \\\"string\\\", \\\"esTypes\\\": [ \\\"_type\\\" ], \\\"searchable\\\": true, \\\"aggregatable\\\": true, \\\"readFromDocValues\\\": false, \\\"metadata_field\\\": true }, { \\\"name\\\": \\\"message\\\", \\\"type\\\": \\\"string\\\", \\\"esTypes\\\": [ \\\"text\\\" ], \\\"searchable\\\": true, \\\"aggregatable\\\": false, \\\"readFromDocValues\\\": false, \\\"metadata_field\\\": false }, { \\\"name\\\": \\\"message.keyword\\\", \\\"type\\\": \\\"string\\\", \\\"esTypes\\\": [ \\\"text\\\" ], \\\"searchable\\\": true, \\\"aggregatable\\\": false, \\\"readFromDocValues\\\": false, \\\"metadata_field\\\": false, \\\"subType\\\": { \\\"multi\\\": { \\\"parent\\\": \\\"message\\\" } } } ]\" + } +} +" \ + "${kibana_api_url_base}/api/saved_objects/index-pattern/${kibana_index_pattern_id}")" +if [[ "${response_status_code}" -ne 200 ]]; then + echo "Failed to create${kibana_index_pattern_title} Kibana index pattern" >&2 + exit 1 +fi diff --git a/fluent-bit-es-cluster/README.md b/fluent-bit-es-cluster/README.md new file mode 100644 index 0000000..402ee0a --- /dev/null +++ b/fluent-bit-es-cluster/README.md @@ -0,0 +1,7 @@ +# Fluent Bit with Elasticsearch output and Upstream Servers + +Refer to: + +1. Fluent Bit documentation for [Upstream Servers](https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/classic-mode/upstream-servers). +1. [https://github.com/fluent/fluent-bit/pull/1560](https://github.com/fluent/fluent-bit/pull/1560). +1. [https://github.com/mabrarov/fluent-bit/compare/master...mabrarov:fluent-bit:feature/out_es_upstream_support](https://github.com/mabrarov/fluent-bit/compare/master...mabrarov:fluent-bit:feature/out_es_upstream_support). diff --git a/fluent-bit-es-cluster/fluent-bit.conf b/fluent-bit-es-cluster/fluent-bit.conf new file mode 100644 index 0000000..d6f0854 --- /dev/null +++ b/fluent-bit-es-cluster/fluent-bit.conf @@ -0,0 +1,17 @@ +[SERVICE] + Flush 10 + +[INPUT] + Name dummy + Dummy { "message" : "this is dummy data" } + +[OUTPUT] + Workers 4 + Name es + Upstream ./out_es_upstream.conf + TLS Off + Retry_Limit 6 + Replace_Dots On + HTTP_User elastic + HTTP_Passwd elastic + Index fluent-bit diff --git a/fluent-bit-es-cluster/upstream.conf b/fluent-bit-es-cluster/upstream.conf new file mode 100644 index 0000000..753e3cb --- /dev/null +++ b/fluent-bit-es-cluster/upstream.conf @@ -0,0 +1,17 @@ +[UPSTREAM] + name elasticsearch-balancing + +[NODE] + name elasticsearch1 + host localhost + port 9201 + +[NODE] + name elasticsearch2 + host localhost + port 9202 + +[NODE] + name elasticsearch3 + host localhost + port 9203 diff --git a/fluent-bit-es/README.md b/fluent-bit-es/README.md new file mode 100644 index 0000000..d9b07b0 --- /dev/null +++ b/fluent-bit-es/README.md @@ -0,0 +1 @@ +# Fluent Bit with Elasticsearch output diff --git a/fluent-bit-es/fluent-bit.conf b/fluent-bit-es/fluent-bit.conf new file mode 100644 index 0000000..340ec40 --- /dev/null +++ b/fluent-bit-es/fluent-bit.conf @@ -0,0 +1,18 @@ +[SERVICE] + Flush 10 + +[INPUT] + Name dummy + Dummy { "message" : "this is dummy data" } + +[OUTPUT] + Workers 1 + Name es + Host localhost + Port 9200 + TLS Off + Retry_Limit 6 + Replace_Dots On + HTTP_User elastic + HTTP_Passwd elastic + Index fluent-bit diff --git a/single-node-elastic-cluster/README.md b/single-node-elastic-cluster/README.md new file mode 100644 index 0000000..480f5d7 --- /dev/null +++ b/single-node-elastic-cluster/README.md @@ -0,0 +1,6 @@ +# Docker Compose project for single-node Elastic cluster + +Docker Compose project for single-node Elastic cluster consisting of: + +1. 1 data / ingesting Elasticsearch node. +1. 1 Kibana node diff --git a/single-node-elastic-cluster/docker-compose.yaml b/single-node-elastic-cluster/docker-compose.yaml new file mode 100644 index 0000000..32559ac --- /dev/null +++ b/single-node-elastic-cluster/docker-compose.yaml @@ -0,0 +1,54 @@ +version: '2.1' + +x-logging: &default-logging + driver: 'json-file' + options: + max-size: '12m' + max-file: '5' + +volumes: + elasticsearch-data: + kibana-data: + +services: + elasticsearch: + image: 'elasticsearch:7.17.9' + mem_limit: '2048m' + mem_reservation: '1024m' + ulimits: + memlock: + soft: -1 + hard: -1 + environment: + discovery.type: 'single-node' + xpack.security.enabled: 'true' + ELASTIC_PASSWORD: 'elastic' + node.attr.data: 'hot' + ports: + - '9200:9200' + - '9300:9300' + volumes: + - 'elasticsearch-data:/usr/share/elasticsearch/data' + logging: *default-logging + + kibana: + image: 'kibana:7.17.9' + mem_limit: '1024m' + mem_reservation: '512m' + ulimits: + memlock: + soft: -1 + hard: -1 + environment: + SERVER_PUBLICBASEURL: 'http://localhost:5601' + ELASTICSEARCH_HOSTS: 'http://elasticsearch:9200' + xpack.security.enabled: 'true' + ELASTICSEARCH_USERNAME: 'elastic' + ELASTICSEARCH_PASSWORD: 'elastic' + TELEMETRY_ENABLED: 'false' + NEWSFEED_ENABLED: 'false' + ports: + - '5601:5601' + volumes: + - 'kibana-data:/usr/share/kibana/data' + logging: *default-logging diff --git a/single-node-elastic-cluster/init.sh b/single-node-elastic-cluster/init.sh new file mode 100644 index 0000000..40ffc5f --- /dev/null +++ b/single-node-elastic-cluster/init.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +set -e + +DOCKER_HOST='localhost' + +elasticsearch_api_user='elastic' +elasticsearch_api_password='elastic' +elasticsearch_api_url_base="http://${DOCKER_HOST}:9200" +kibana_api_url_base="http://${DOCKER_HOST}:5601" +elasticsearch_index_alias="fluent-bit" +elasticsearch_index_name="${elasticsearch_index_alias}-000001" +kibana_index_pattern_id="${elasticsearch_index_alias}" +kibana_index_pattern_title="${elasticsearch_index_alias}-*" + +response_status_code="$(curl -ks -X POST \ + -o /dev/null -w '%{http_code}\n' \ + --user "${elasticsearch_api_user}:${elasticsearch_api_password}" \ + "${elasticsearch_api_url_base}/_license/start_trial?acknowledge=true")" +if [[ "${response_status_code}" -ne 200 ]]; then + echo "Failed to set Elasticsearch trial license" >&2 + exit 1 +fi + +response_status_code="$(curl -s -X PUT \ + -o /dev/null -w '%{http_code}\n' \ + --user "${elasticsearch_api_user}:${elasticsearch_api_password}" \ + -H 'Content-Type: application/json' \ + -d \ +"{ + \"aliases\": { + \"${elasticsearch_index_alias}\": {} + }, + \"mappings\": { + \"properties\": { + \"@timestamp\": { + \"type\": \"date\" + }, + \"message\": { + \"type\": \"text\", + \"fields\": { + \"keyword\": { + \"type\": \"text\" + } + } + } + } + }, + \"settings\": { + \"index\": { + \"number_of_shards\": 1, + \"number_of_replicas\": 0, + \"refresh_interval\": \"30s\" + } + } +} +" \ + "${elasticsearch_api_url_base}/${elasticsearch_index_name}")" +if [[ "${response_status_code}" -ne 200 ]]; then + echo "Failed to create ${elasticsearch_index_name} Elasticsearch index" >&2 + exit 1 +fi + +response_status_code="$(curl -s -X POST \ + -o /dev/null -w '%{http_code}\n' \ + --user "${elasticsearch_api_user}:${elasticsearch_api_password}" \ + -H 'kbn-xsrf: required_header' \ + -H 'Content-Type: application/json' \ + -d \ +"{ + \"attributes\": { + \"title\": \"${kibana_index_pattern_title}\", + \"timeFieldName\": \"@timestamp\", + \"fields\": \"[ { \\\"name\\\": \\\"@timestamp\\\", \\\"type\\\": \\\"date\\\", \\\"esTypes\\\": [ \\\"date\\\" ], \\\"searchable\\\": true, \\\"aggregatable\\\": true, \\\"readFromDocValues\\\": true, \\\"metadata_field\\\": false }, { \\\"name\\\": \\\"_id\\\", \\\"type\\\": \\\"string\\\", \\\"esTypes\\\": [ \\\"_id\\\" ], \\\"searchable\\\": true, \\\"aggregatable\\\": true, \\\"readFromDocValues\\\": false, \\\"metadata_field\\\": true }, { \\\"name\\\": \\\"_index\\\", \\\"type\\\": \\\"string\\\", \\\"esTypes\\\": [ \\\"_index\\\" ], \\\"searchable\\\": true, \\\"aggregatable\\\": true, \\\"readFromDocValues\\\": false, \\\"metadata_field\\\": true }, { \\\"name\\\": \\\"_score\\\", \\\"type\\\": \\\"number\\\", \\\"searchable\\\": false, \\\"aggregatable\\\": false, \\\"readFromDocValues\\\": false, \\\"metadata_field\\\": true }, { \\\"name\\\": \\\"_source\\\", \\\"type\\\": \\\"_source\\\", \\\"esTypes\\\": [ \\\"_source\\\" ], \\\"searchable\\\": false, \\\"aggregatable\\\": false, \\\"readFromDocValues\\\": false, \\\"metadata_field\\\": true }, { \\\"name\\\": \\\"_type\\\", \\\"type\\\": \\\"string\\\", \\\"esTypes\\\": [ \\\"_type\\\" ], \\\"searchable\\\": true, \\\"aggregatable\\\": true, \\\"readFromDocValues\\\": false, \\\"metadata_field\\\": true }, { \\\"name\\\": \\\"message\\\", \\\"type\\\": \\\"string\\\", \\\"esTypes\\\": [ \\\"text\\\" ], \\\"searchable\\\": true, \\\"aggregatable\\\": false, \\\"readFromDocValues\\\": false, \\\"metadata_field\\\": false }, { \\\"name\\\": \\\"message.keyword\\\", \\\"type\\\": \\\"string\\\", \\\"esTypes\\\": [ \\\"text\\\" ], \\\"searchable\\\": true, \\\"aggregatable\\\": false, \\\"readFromDocValues\\\": false, \\\"metadata_field\\\": false, \\\"subType\\\": { \\\"multi\\\": { \\\"parent\\\": \\\"message\\\" } } } ]\" + } +} +" \ + "${kibana_api_url_base}/api/saved_objects/index-pattern/${kibana_index_pattern_id}")" +if [[ "${response_status_code}" -ne 200 ]]; then + echo "Failed to create${kibana_index_pattern_title} Kibana index pattern" >&2 + exit 1 +fi