Configuration for testing fluent/fluent-bit#1560.

.gitattributes

@@ -0,0 +1,6 @@
+* text=auto
+*.sh text eol=lf
+*.conf text eol=lf
+*.bat text eol=crlf
+*.cmd text eol=crlf
+*.ps1 text eol=crlf

.gitignore

@@ -0,0 +1,2 @@
+/.idea/
+/build/

README.md

@@ -0,0 +1 @@
+# Configurations for Elastic stack and related technologies

elastic-cluster/README.md

@@ -0,0 +1,7 @@
+# Docker Compose project for Elastic cluster
+
+Docker Compose project for Elastic cluster consisting of:
+
+1. 3 master / data / ingesting Elasticsearch nodes.
+1. 1 coordinating Elasticsearch node.
+1. 1 Kibana node

elastic-cluster/docker-compose.yaml

@@ -0,0 +1,158 @@
+version: '2.1'
+
+x-logging: &default-logging
+  driver: 'json-file'
+  options:
+    max-size: '12m'
+    max-file: '5'
+
+volumes:
+  elasticsearch1-data:
+  elasticsearch2-data:
+  elasticsearch3-data:
+  elasticsearch4-data:
+  kibana-data:
+
+services:
+  elasticsearch1:
+    image: 'elasticsearch:7.17.9'
+    mem_limit: '1024m'
+    mem_reservation: '512m'
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    environment:
+      node.name: 'elasticsearch1'
+      cluster.name: 'elasticsearch'
+      discovery.seed_hosts: 'elasticsearch1,elasticsearch2,elasticsearch3'
+      cluster.initial_master_nodes: 'elasticsearch1,elasticsearch2,elasticsearch3'
+      node.master: 'true'
+      node.data: 'true'
+      node.attr.data: 'hot'
+      node.ingest: 'true'
+      node.ml: 'false'
+      node.transform: 'false'
+      node.remote_cluster_client: 'false'
+      xpack.ml.enabled: 'false'
+      xpack.transform.enabled: 'false'
+      xpack.security.enabled: 'true'
+      ELASTIC_PASSWORD: 'elastic'
+    ports:
+      - '9201:9200'
+    volumes:
+      - 'elasticsearch1-data:/usr/share/elasticsearch/data'
+    logging: *default-logging
+
+  elasticsearch2:
+    image: 'elasticsearch:7.17.9'
+    mem_limit: '1024m'
+    mem_reservation: '512m'
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    environment:
+      node.name: 'elasticsearch2'
+      cluster.name: 'elasticsearch'
+      discovery.seed_hosts: 'elasticsearch1,elasticsearch2,elasticsearch3'
+      cluster.initial_master_nodes: 'elasticsearch1,elasticsearch2,elasticsearch3'
+      bootstrap.memory_lock: 'true'
+      node.master: 'true'
+      node.data: 'true'
+      node.attr.data: 'hot'
+      node.ingest: 'true'
+      node.ml: 'false'
+      node.transform: 'false'
+      node.remote_cluster_client: 'false'
+      xpack.ml.enabled: 'false'
+      xpack.transform.enabled: 'false'
+      xpack.security.enabled: 'true'
+      ELASTIC_PASSWORD: 'elastic'
+    ports:
+      - '9202:9200'
+    volumes:
+      - 'elasticsearch2-data:/usr/share/elasticsearch/data'
+    logging: *default-logging
+
+  elasticsearch3:
+    image: 'elasticsearch:7.17.9'
+    mem_limit: '1024m'
+    mem_reservation: '512m'
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    environment:
+      node.name: 'elasticsearch3'
+      cluster.name: 'elasticsearch'
+      discovery.seed_hosts: 'elasticsearch1,elasticsearch2,elasticsearch3'
+      cluster.initial_master_nodes: 'elasticsearch1,elasticsearch2,elasticsearch3'
+      bootstrap.memory_lock: 'true'
+      node.master: 'true'
+      node.data: 'true'
+      node.attr.data: 'hot'
+      node.ingest: 'true'
+      node.ml: 'false'
+      node.transform: 'false'
+      node.remote_cluster_client: 'false'
+      xpack.ml.enabled: 'false'
+      xpack.transform.enabled: 'false'
+      xpack.security.enabled: 'true'
+      ELASTIC_PASSWORD: 'elastic'
+    ports:
+      - '9203:9200'
+    volumes:
+      - 'elasticsearch3-data:/usr/share/elasticsearch/data'
+    logging: *default-logging
+
+  elasticsearch4:
+    image: 'elasticsearch:7.17.9'
+    mem_limit: '1024m'
+    mem_reservation: '512m'
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    environment:
+      node.name: 'elasticsearch4'
+      cluster.name: 'elasticsearch'
+      discovery.seed_hosts: 'elasticsearch1,elasticsearch2,elasticsearch3'
+      cluster.initial_master_nodes: 'elasticsearch1,elasticsearch2,elasticsearch3'
+      bootstrap.memory_lock: 'true'
+      node.master: 'false'
+      node.voting_only: 'false'
+      node.data: 'false'
+      node.ingest: 'false'
+      node.ml: 'false'
+      node.transform: 'false'
+      node.remote_cluster_client: 'false'
+      xpack.ml.enabled: 'false'
+      xpack.transform.enabled: 'false'
+      xpack.security.enabled: 'true'
+      ELASTIC_PASSWORD: 'elastic'
+    volumes:
+      - 'elasticsearch4-data:/usr/share/elasticsearch/data'
+    logging: *default-logging
+
+  kibana:
+    image: 'kibana:7.17.9'
+    mem_limit: '1024m'
+    mem_reservation: '512m'
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    environment:
+      SERVER_PUBLICBASEURL: 'http://localhost:5601'
+      ELASTICSEARCH_HOSTS: 'http://elasticsearch4:9200'
+      xpack.security.enabled: 'true'
+      ELASTICSEARCH_USERNAME: 'elastic'
+      ELASTICSEARCH_PASSWORD: 'elastic'
+      TELEMETRY_ENABLED: 'false'
+      NEWSFEED_ENABLED: 'false'
+    ports:
+      - '5601:5601'
+    volumes:
+      - 'kibana-data:/usr/share/kibana/data'
+    logging: *default-logging

elastic-cluster/init.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+set -e
+
+DOCKER_HOST='localhost'
+
+elasticsearch_api_user='elastic'
+elasticsearch_api_password='elastic'
+elasticsearch_api_url_base="http://${DOCKER_HOST}:9201"
+kibana_api_url_base="http://${DOCKER_HOST}:5601"
+elasticsearch_index_alias="fluent-bit"
+elasticsearch_index_name="${elasticsearch_index_alias}-000001"
+kibana_index_pattern_id="${elasticsearch_index_alias}"
+kibana_index_pattern_title="${elasticsearch_index_alias}-*"
+
+response_status_code="$(curl -ks -X POST \
+  -o /dev/null -w '%{http_code}\n' \
+  --user "${elasticsearch_api_user}:${elasticsearch_api_password}" \
+  "${elasticsearch_api_url_base}/_license/start_trial?acknowledge=true")"
+if [[ "${response_status_code}" -ne 200 ]]; then
+  echo "Failed to set Elasticsearch trial license" >&2
+  exit 1
+fi
+
+response_status_code="$(curl -s -X PUT \
+  -o /dev/null -w '%{http_code}\n' \
+  --user "${elasticsearch_api_user}:${elasticsearch_api_password}" \
+  -H 'Content-Type: application/json' \
+  -d \
+"{
+  \"aliases\": {
+    \"${elasticsearch_index_alias}\": {}
+  },
+  \"mappings\": {
+    \"properties\": {
+      \"@timestamp\": {
+        \"type\": \"date\"
+      },
+      \"message\": {
+        \"type\": \"text\",
+        \"fields\": {
+          \"keyword\": {
+            \"type\": \"text\"
+          }
+        }
+      }
+    }
+  },
+  \"settings\": {
+    \"index\": {
+      \"number_of_shards\": 2,
+      \"number_of_replicas\": 1,
+      \"refresh_interval\": \"30s\"
+    }
+  }
+}
+" \
+  "${elasticsearch_api_url_base}/${elasticsearch_index_name}")"
+if [[ "${response_status_code}" -ne 200 ]]; then
+  echo "Failed to create ${elasticsearch_index_name} Elasticsearch index" >&2
+  exit 1
+fi
+
+response_status_code="$(curl -s -X POST \
+  -o /dev/null -w '%{http_code}\n' \
+  --user "${elasticsearch_api_user}:${elasticsearch_api_password}" \
+  -H 'kbn-xsrf: required_header' \
+  -H 'Content-Type: application/json' \
+  -d \
+"{
+  \"attributes\": {
+    \"title\": \"${kibana_index_pattern_title}\",
+    \"timeFieldName\": \"@timestamp\",
+    \"fields\": \"[ { \\\"name\\\": \\\"@timestamp\\\", \\\"type\\\": \\\"date\\\", \\\"esTypes\\\": [ \\\"date\\\" ], \\\"searchable\\\": true, \\\"aggregatable\\\": true, \\\"readFromDocValues\\\": true, \\\"metadata_field\\\": false }, { \\\"name\\\": \\\"_id\\\", \\\"type\\\": \\\"string\\\", \\\"esTypes\\\": [ \\\"_id\\\" ], \\\"searchable\\\": true, \\\"aggregatable\\\": true, \\\"readFromDocValues\\\": false, \\\"metadata_field\\\": true }, { \\\"name\\\": \\\"_index\\\", \\\"type\\\": \\\"string\\\", \\\"esTypes\\\": [ \\\"_index\\\" ], \\\"searchable\\\": true, \\\"aggregatable\\\": true, \\\"readFromDocValues\\\": false, \\\"metadata_field\\\": true }, { \\\"name\\\": \\\"_score\\\", \\\"type\\\": \\\"number\\\", \\\"searchable\\\": false, \\\"aggregatable\\\": false, \\\"readFromDocValues\\\": false, \\\"metadata_field\\\": true }, { \\\"name\\\": \\\"_source\\\", \\\"type\\\": \\\"_source\\\", \\\"esTypes\\\": [ \\\"_source\\\" ], \\\"searchable\\\": false, \\\"aggregatable\\\": false, \\\"readFromDocValues\\\": false, \\\"metadata_field\\\": true }, { \\\"name\\\": \\\"_type\\\", \\\"type\\\": \\\"string\\\", \\\"esTypes\\\": [ \\\"_type\\\" ], \\\"searchable\\\": true, \\\"aggregatable\\\": true, \\\"readFromDocValues\\\": false, \\\"metadata_field\\\": true }, { \\\"name\\\": \\\"message\\\", \\\"type\\\": \\\"string\\\", \\\"esTypes\\\": [ \\\"text\\\" ], \\\"searchable\\\": true, \\\"aggregatable\\\": false, \\\"readFromDocValues\\\": false, \\\"metadata_field\\\": false }, { \\\"name\\\": \\\"message.keyword\\\", \\\"type\\\": \\\"string\\\", \\\"esTypes\\\": [ \\\"text\\\" ], \\\"searchable\\\": true, \\\"aggregatable\\\": false, \\\"readFromDocValues\\\": false, \\\"metadata_field\\\": false, \\\"subType\\\": { \\\"multi\\\": { \\\"parent\\\": \\\"message\\\" } } } ]\"
+  }
+}
+" \
+  "${kibana_api_url_base}/api/saved_objects/index-pattern/${kibana_index_pattern_id}")"
+if [[ "${response_status_code}" -ne 200 ]]; then
+  echo "Failed to create${kibana_index_pattern_title} Kibana index pattern" >&2
+  exit 1
+fi

fluent-bit-es-cluster/README.md

@@ -0,0 +1,7 @@
+# Fluent Bit with Elasticsearch output and Upstream Servers
+
+Refer to:
+
+1. Fluent Bit documentation for [Upstream Servers](https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/classic-mode/upstream-servers).
+1. [https://github.com/fluent/fluent-bit/pull/1560](https://github.com/fluent/fluent-bit/pull/1560).
+1. [https://github.com/mabrarov/fluent-bit/compare/master...mabrarov:fluent-bit:feature/out_es_upstream_support](https://github.com/mabrarov/fluent-bit/compare/master...mabrarov:fluent-bit:feature/out_es_upstream_support).

fluent-bit-es-cluster/fluent-bit.conf

@@ -0,0 +1,17 @@
+[SERVICE]
+    Flush          10
+
+[INPUT]
+    Name dummy
+    Dummy { "message" : "this is dummy data" }
+
+[OUTPUT]
+    Workers         4
+    Name            es
+    Upstream        ./out_es_upstream.conf
+    TLS             Off
+    Retry_Limit     6
+    Replace_Dots    On
+    HTTP_User       elastic
+    HTTP_Passwd     elastic
+    Index           fluent-bit

fluent-bit-es-cluster/upstream.conf

@@ -0,0 +1,17 @@
+[UPSTREAM]
+    name       elasticsearch-balancing
+
+[NODE]
+    name       elasticsearch1
+    host       localhost
+    port       9201
+
+[NODE]
+    name       elasticsearch2
+    host       localhost
+    port       9202
+
+[NODE]
+    name       elasticsearch3
+    host       localhost
+    port       9203

fluent-bit-es/README.md

@@ -0,0 +1 @@
+# Fluent Bit with Elasticsearch output

fluent-bit-es/fluent-bit.conf

@@ -0,0 +1,18 @@
+[SERVICE]
+    Flush           10
+
+[INPUT]
+    Name dummy
+    Dummy { "message" : "this is dummy data" }
+
+[OUTPUT]
+    Workers         1
+    Name            es
+    Host            localhost
+    Port            9200
+    TLS             Off
+    Retry_Limit     6
+    Replace_Dots    On
+    HTTP_User       elastic
+    HTTP_Passwd     elastic
+    Index           fluent-bit

single-node-elastic-cluster/README.md

@@ -0,0 +1,6 @@
+# Docker Compose project for single-node Elastic cluster
+
+Docker Compose project for single-node Elastic cluster consisting of:
+
+1. 1 data / ingesting Elasticsearch node.
+1. 1 Kibana node

single-node-elastic-cluster/docker-compose.yaml

@@ -0,0 +1,54 @@
+version: '2.1'
+
+x-logging: &default-logging
+  driver: 'json-file'
+  options:
+    max-size: '12m'
+    max-file: '5'
+
+volumes:
+  elasticsearch-data:
+  kibana-data:
+
+services:
+  elasticsearch:
+    image: 'elasticsearch:7.17.9'
+    mem_limit: '2048m'
+    mem_reservation: '1024m'
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    environment:
+      discovery.type: 'single-node'
+      xpack.security.enabled: 'true'
+      ELASTIC_PASSWORD: 'elastic'
+      node.attr.data: 'hot'
+    ports:
+      - '9200:9200'
+      - '9300:9300'
+    volumes:
+      - 'elasticsearch-data:/usr/share/elasticsearch/data'
+    logging: *default-logging
+
+  kibana:
+    image: 'kibana:7.17.9'
+    mem_limit: '1024m'
+    mem_reservation: '512m'
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    environment:
+      SERVER_PUBLICBASEURL: 'http://localhost:5601'
+      ELASTICSEARCH_HOSTS: 'http://elasticsearch:9200'
+      xpack.security.enabled: 'true'
+      ELASTICSEARCH_USERNAME: 'elastic'
+      ELASTICSEARCH_PASSWORD: 'elastic'
+      TELEMETRY_ENABLED: 'false'
+      NEWSFEED_ENABLED: 'false'
+    ports:
+      - '5601:5601'
+    volumes:
+      - 'kibana-data:/usr/share/kibana/data'
+    logging: *default-logging