Merge branch 'fluent:master' into timestamp-encode-decode-big-endian

CMakeLists.txt

@@ -88,6 +88,7 @@ elseif(FLB_SYSTEM_FREEBSD)
   set(WAMR_DISABLE_STACK_HW_BOUND_CHECK 1)
 endif()
 
+INCLUDE(TestBigEndian)
 include(GNUInstallDirs)
 include(ExternalProject)
 include(cmake/FindJournald.cmake)
@@ -772,6 +773,15 @@ if(FLB_HAVE_UNIX_SOCKET)
   FLB_DEFINITION(FLB_HAVE_UNIX_SOCKET)
 endif()
 
+# byte order detection
+test_big_endian(BIG_ENDIAN_SYSTEM_DETECTED)
+
+if (BIG_ENDIAN_SYSTEM_DETECTED)
+    FLB_DEFINITION(FLB_HAVE_BIG_ENDIAN_SYSTEM)
+else()
+    FLB_DEFINITION(FLB_HAVE_LITTLE_ENDIAN_SYSTEM)
+endif ()
+
 # Configuration file YAML format support
 if(FLB_CONFIG_YAML)
   find_package(PkgConfig)

include/fluent-bit/flb_config_map.h

@@ -55,7 +55,7 @@
 struct flb_config_map_val {
     union {
         int i_num;                    /* FLB_CONFIG_MAP_INT */
-        char boolean;                 /* FLB_CONFIG_MAP_BOOL */
+        int boolean;                  /* FLB_CONFIG_MAP_BOOL */
         double d_num;                 /* FLB_CONFIG_MAP_DOUBLE */
         size_t s_num;                 /* FLB_CONFIG_MAP_SIZE */
         flb_sds_t str;                /* FLB_CONFIG_MAP_STR */

include/fluent-bit/flb_endian.h

@@ -55,9 +55,7 @@
 #define FLB_BIG_ENDIAN    1
 
 #ifndef FLB_BYTE_ORDER
-    #if defined(__BYTE_ORDER__) &&  __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-        #define FLB_BYTE_ORDER FLB_BIG_ENDIAN
-    #elif defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || defined(_BIG_ENDIAN)
+    #ifdef FLB_HAVE_BIG_ENDIAN_SYSTEM
         #define FLB_BYTE_ORDER FLB_BIG_ENDIAN
     #else
         #define FLB_BYTE_ORDER FLB_LITTLE_ENDIAN

plugins/filter_log_to_metrics/log_to_metrics.c

@@ -463,6 +463,7 @@ static int cb_log_to_metrics_init(struct flb_filter_instance *f_ins,
     char value_field[MAX_METRIC_LENGTH];
     struct flb_input_instance *input_ins;
 
+
     int i;
     /* Create context */
     ctx = flb_calloc(1, sizeof(struct log_to_metrics_ctx));
@@ -551,15 +552,17 @@ static int cb_log_to_metrics_init(struct flb_filter_instance *f_ins,
 
     /* Check property metric mode */
     ctx->mode = 0;
-    tmp = (char *) flb_filter_get_property("metric_mode", f_ins);
-    if (tmp != NULL) {
-        if (strcasecmp(tmp, FLB_LOG_TO_METRICS_COUNTER_STR) == 0) {
+    if (ctx->mode_name != NULL) {
+        if (strcasecmp(ctx->mode_name,
+                       FLB_LOG_TO_METRICS_COUNTER_STR) == 0) {
             ctx->mode = FLB_LOG_TO_METRICS_COUNTER;
         }
-        else if (strcasecmp(tmp, FLB_LOG_TO_METRICS_GAUGE_STR) == 0) {
+        else if (strcasecmp(ctx->mode_name,
+                            FLB_LOG_TO_METRICS_GAUGE_STR) == 0) {
             ctx->mode = FLB_LOG_TO_METRICS_GAUGE;
         }
-        else if (strcasecmp(tmp, FLB_LOG_TO_METRICS_HISTOGRAM_STR) == 0) {
+        else if (strcasecmp(ctx->mode_name,
+                            FLB_LOG_TO_METRICS_HISTOGRAM_STR) == 0) {
             ctx->mode = FLB_LOG_TO_METRICS_HISTOGRAM;
         }
         else {
@@ -589,7 +592,7 @@ static int cb_log_to_metrics_init(struct flb_filter_instance *f_ins,
     /* Check property subsystem name */
     if (ctx->metric_subsystem == NULL || strlen(ctx->metric_subsystem) == 0) {
         snprintf(metric_subsystem, sizeof(metric_subsystem) - 1, "%s",
-                 tmp);
+                 ctx->mode_name);
     }
     else {
         snprintf(metric_subsystem, sizeof(metric_subsystem) - 1, "%s",
@@ -977,7 +980,7 @@ static struct flb_config_map config_map[] = {
     },
     {
      FLB_CONFIG_MAP_STR, "metric_mode", "counter",
-     0, FLB_TRUE, offsetof(struct log_to_metrics_ctx, mode),
+     0, FLB_TRUE, offsetof(struct log_to_metrics_ctx, mode_name),
      "Mode selector. Values counter, gauge,"
      " or histogram. Summary is not supported"
     },

plugins/filter_log_to_metrics/log_to_metrics.h

@@ -73,6 +73,7 @@ struct log_to_metrics_ctx {
 
     /* config options */
     int mode;
+    flb_sds_t mode_name;
     int discard_logs;
     int kubernetes_mode;
     flb_sds_t metric_name;

plugins/in_emitter/emitter.c

@@ -312,6 +312,7 @@ static int cb_emitter_init(struct flb_input_instance *in,
 
     ret = flb_input_config_map_set(in, (void *) ctx);
     if (ret == -1) {
+        flb_free(ctx);
         return -1;
     }
 

plugins/in_process_exporter_metrics/pe_process.c

@@ -276,7 +276,7 @@ static void reset_proc_state(struct proc_state *state) {
 static int check_path_for_proc(struct flb_pe *ctx, const char *prefix, const char *path)
 {
     int len;
-    flb_sds_t p;
+    flb_sds_t p = NULL;
 
     /* Compose the proc path */
     p = flb_sds_create(prefix);
@@ -303,8 +303,8 @@ static int check_path_for_proc(struct flb_pe *ctx, const char *prefix, const cha
 
 static int get_name(const char *entry, char **out_name, char *id_entry)
 {
-    flb_sds_t tmp;
-    flb_sds_t tmp_name;
+    flb_sds_t tmp = NULL;
+    flb_sds_t tmp_name = NULL;
 
     tmp = strdup(entry);
     tmp_name = strtok(tmp, ")");
@@ -324,8 +324,8 @@ static int process_proc_thread_io(struct flb_pe *ctx, uint64_t ts,
                                   struct flb_slist_entry *thread)
 {
     int ret;
-    flb_sds_t tmp;
-    flb_sds_t status;
+    flb_sds_t tmp = NULL;
+    flb_sds_t status = NULL;
     uint64_t val;
     struct mk_list io_list;
     struct mk_list *ihead;
@@ -382,9 +382,9 @@ static int process_proc_thread_status(struct flb_pe *ctx, uint64_t ts,
                                       struct flb_slist_entry *thread)
 {
     int ret;
-    flb_sds_t tmp;
-    flb_sds_t name;
-    flb_sds_t status;
+    flb_sds_t tmp = NULL;
+    flb_sds_t name = NULL;
+    flb_sds_t status = NULL;
     uint64_t val;
     struct mk_list status_list;
     struct mk_list *shead;
@@ -481,9 +481,9 @@ static int process_proc_thread_status(struct flb_pe *ctx, uint64_t ts,
 static int process_thread_update(struct flb_pe *ctx, uint64_t ts, flb_sds_t pid, flb_sds_t name)
 {
     int ret;
-    flb_sds_t tmp;
-    flb_sds_t thread_name;
-    flb_sds_t tid_str;
+    flb_sds_t tmp = NULL;
+    flb_sds_t thread_name = NULL;
+    flb_sds_t tid_str = NULL;
     uint64_t val;
     const char *pattern = "/[0-9]*";
     struct mk_list *head;
@@ -648,8 +648,8 @@ static int process_proc_io(struct flb_pe *ctx, uint64_t ts,
                            struct flb_slist_entry *process)
 {
     int ret;
-    flb_sds_t tmp;
-    flb_sds_t status;
+    flb_sds_t tmp = NULL;
+    flb_sds_t status = NULL;
     uint64_t val;
     struct mk_list io_list;
     struct mk_list *ihead;
@@ -784,9 +784,9 @@ static int process_proc_fds(struct flb_pe *ctx, uint64_t ts,
 static int process_proc_status(struct flb_pe *ctx, uint64_t ts, flb_sds_t pid, struct flb_slist_entry *process)
 {
     int ret;
-    flb_sds_t tmp;
-    flb_sds_t name;
-    flb_sds_t status;
+    flb_sds_t tmp = NULL;
+    flb_sds_t name = NULL;
+    flb_sds_t status = NULL;
     uint64_t val;
     struct mk_list status_list;
     struct mk_list *shead;
@@ -881,8 +881,8 @@ static int process_proc_status(struct flb_pe *ctx, uint64_t ts, flb_sds_t pid, s
 static int process_proc_boot_time(struct flb_pe *ctx, uint64_t *out_boot_time)
 {
     int ret;
-    flb_sds_t tmp;
-    flb_sds_t status;
+    flb_sds_t tmp = NULL;
+    flb_sds_t status = NULL;
     uint64_t val;
     struct mk_list stat_list;
     struct mk_list *rshead;
@@ -923,12 +923,12 @@ static int process_proc_boot_time(struct flb_pe *ctx, uint64_t *out_boot_time)
 static int process_update(struct flb_pe *ctx)
 {
     int ret;
-    flb_sds_t tmp;
-    flb_sds_t name;
-    flb_sds_t pid_str;
-    flb_sds_t state_str;
-    flb_sds_t ppid_str;
-    flb_sds_t thread_str;
+    flb_sds_t tmp = NULL;
+    flb_sds_t name = NULL;
+    flb_sds_t pid_str = NULL;
+    flb_sds_t state_str = NULL;
+    flb_sds_t ppid_str = NULL;
+    flb_sds_t thread_str = NULL;
     struct mk_list *head;
     struct mk_list *ehead;
     struct mk_list procfs_list;

plugins/in_winevtlog/pack.c

@@ -277,6 +277,15 @@ static int pack_sid(struct winevtlog_config *ctx, PSID sid, int extract_sid)
 
     if (ConvertSidToStringSidW(sid, &wide_sid)) {
         if (extract_sid == FLB_TRUE) {
+            /* Skip to translate SID for capability SIDs.
+             * ref: https://learn.microsoft.com/en-us/windows-server/identity/ad-ds/manage/understand-security-identifiers
+             * See also: https://learn.microsoft.com/en-us/troubleshoot/windows-server/windows-security/sids-not-resolve-into-friendly-names
+             */
+            if (wcsnicmp(wide_sid, L"S-1-15-3-", 9) == 0) {
+                flb_plg_debug(ctx->ins, "This SID is one of the capability SIDs. Skip.");
+
+                goto not_mapped_error;
+            }
             if (!LookupAccountSidA(NULL, sid,
                                    account, &len, domain,
                                    &len, &sid_type)) {

src/flb_upstream_ha.c

@@ -27,6 +27,7 @@
 #include <fluent-bit/flb_upstream_node.h>
 #include <fluent-bit/flb_config_format.h>
 #include <fluent-bit/flb_kv.h>
+#include <fluent-bit/flb_env.h>
 
 #include <ctype.h>
 #include <sys/types.h>
@@ -108,6 +109,25 @@ struct flb_upstream_node *flb_upstream_ha_node_get(struct flb_upstream_ha *ctx)
     return node;
 }
 
+static inline flb_sds_t translate_environment_variables(flb_sds_t *value,
+                                                        struct flb_config *config,
+                                                        int in_place_operation)
+{
+    flb_sds_t result;
+
+    result = flb_env_var_translate(config->env, *value);
+
+    if (result != NULL) {
+        if (in_place_operation) {
+            flb_sds_destroy(*value);
+
+            *value = (flb_sds_t) result;
+        }
+    }
+
+    return result;
+}
+
 static struct flb_upstream_node *create_node(int id,
                                              struct flb_cf *cf,
                                              struct flb_cf_section *s,
@@ -133,6 +153,7 @@ static struct flb_upstream_node *create_node(int id,
     char *tls_crt_file = NULL;
     char *tls_key_file = NULL;
     char *tls_key_passwd = NULL;
+    flb_sds_t translated_value;
     struct cfl_list *head;
     struct cfl_kvpair *entry;
     struct flb_hash_table *ht;
@@ -214,6 +235,16 @@ static struct flb_upstream_node *create_node(int id,
     /* tls.key_file */
     tls_key_passwd = flb_cf_section_property_get_string(cf, s, "tls.key_passwd");
 
+    translate_environment_variables((flb_sds_t *) &name, config, FLB_TRUE);
+    translate_environment_variables((flb_sds_t *) &host, config, FLB_TRUE);
+    translate_environment_variables((flb_sds_t *) &port, config, FLB_TRUE);
+    translate_environment_variables((flb_sds_t *) &tls_vhost, config, FLB_TRUE);
+    translate_environment_variables((flb_sds_t *) &tls_ca_path, config, FLB_TRUE);
+    translate_environment_variables((flb_sds_t *) &tls_ca_file, config, FLB_TRUE);
+    translate_environment_variables((flb_sds_t *) &tls_crt_file, config, FLB_TRUE);
+    translate_environment_variables((flb_sds_t *) &tls_key_file, config, FLB_TRUE);
+    translate_environment_variables((flb_sds_t *) &tls_key_passwd, config, FLB_TRUE);
+
     /*
      * Create hash table to store unknown key/values that might be used
      * by the caller plugin.
@@ -252,12 +283,39 @@ static struct flb_upstream_node *create_node(int id,
         }
         key[klen] = '\0';
 
+        translated_value = translate_environment_variables(
+                            (flb_sds_t *) &entry->val->data.as_string,
+                            config,
+                            FLB_FALSE);
+
+        if (translated_value == NULL) {
+            flb_error("[upstream_ha] cannot perform environment variable "
+                      "lookup for key %s",
+                      entry->key);
+            flb_hash_table_destroy(ht);
+
+            return NULL;
+        }
+        vlen = flb_sds_len(translated_value);
+
+        /* We need to ensure that vlen is larger than zero in order for
+         * to store a copy of the value instead of a reference but this
+         * is not a problem because flb_sds_t instances always have at
+         * least the NULL terminator byte.
+         */
+
+        if (vlen == 0) {
+            vlen = 1;
+        }
+
         /* Add the key and value to the hash table */
-        ret = flb_hash_table_add(ht, key, klen, entry->val->data.as_string, vlen);
+        ret = flb_hash_table_add(ht, key, klen, translated_value, vlen);
         if (ret == -1) {
             flb_error("[upstream_ha] cannot add key %s to hash table",
                       entry->key);
         }
+
+        flb_sds_destroy(translated_value);
     }
 
     node = flb_upstream_node_create(name, host, port, tls, tls_verify,