POD metadata is not extracted to sourceName

[danil-fomichev]

Here is what we see in SumoLogic logs search for all applications:

Name: %{namespace}.%{pod}.%{container}
Category: kubernetes/%{namespace}/%{pod_name} 

is there something wrong with configuration file?

<ROOT>
  <match containers.**.fluentd**>
    @type null
  </match>
  <source>
    @type monitor_agent
    bind "0.0.0.0"
    port 24220
  </source>
  <source>
    @type tail
    format json
    time_key time
    path "/var/lib/docker/containers/*/*.log"
    exclude_path
    pos_file "/mnt/pos/ggcp-containers.log.pos"
    time_format %Y-%m-%dT%H:%M:%S.%NZ
    tag "containers.*"
    read_from_head true
    enable_stat_watcher true
    <parse>
      time_key time
      time_format %Y-%m-%dT%H:%M:%S.%NZ
      @type json
      time_type string
    </parse>
  </source>
  <filter containers.**>
    @type concat
    key "log"
    multiline_start_regexp "/^\\w{3} \\d{1,2}, \\d{4}/"
    separator ""
    timeout_label "@NORMAL"
  </filter>
  <match containers.**>
    @type relabel
    @label @NORMAL
  </match>
  <label @NORMAL>
    <filter containers.**>
      @type kubernetes_metadata
      @log_level "warn"
      annotation_match ["sumologic.com.*"]
      de_dot false
      watch true
      ca_file ""
      verify_ssl true
      client_cert ""
      client_key ""
      bearer_token_file ""
      cache_size 1000
      cache_ttl 3600
      tag_to_kubernetes_name_regexp ".+?\\.containers\\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\\.log$"
      merge_json_log false
    </filter>
    <filter containers.**>
      @type kubernetes_sumologic
      source_name "%{namespace}.%{pod}.%{container}"
      source_host ""
      log_format "text"
      kubernetes_meta true
      kubernetes_meta_reduce false
      add_stream true
      add_time true
      source_category "%{namespace}/%{pod_name}"
      source_category_prefix "kubernetes/"
      source_category_replace_dash "/"
      exclude_namespace_regex "/^(kube-system|monitoring)$/"
      exclude_pod_regex ""
      exclude_container_regex ""
      exclude_host_regex ""
    </filter>
    <match **>
      @type sumologic
      log_key "log"
      endpoint xxxxxx
      verify_ssl true
      log_format "text"
      flush_interval 60
      num_threads 1
      open_timeout 60
      add_timestamp true
      timestamp_key "timestamp"
      proxy_uri ""
      <buffer>
        flush_thread_count 1
        flush_interval 60
      </buffer>
    </match>
  </label>
  <source>
    @type tail
    format /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
    time_format %Y-%m-%dT%H:%M:%S.%NZ
    path "/var/lib/docker.log"
    exclude_path
    pos_file "/mnt/pos/ggcp-docker.log.pos"
    tag "docker"
    enable_stat_watcher true
    <parse>
      time_format %Y-%m-%dT%H:%M:%S.%NZ
      @type regexp
      expression ^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?
    </parse>
  </source>
  <filter docker.**>
    @type kubernetes_sumologic
    source_category "docker"
    source_name "k8s_docker"
    source_category_prefix "kubernetes/"
  </filter>
  <source>
    @type tail
    format /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/
    time_format %Y-%m-%d %H:%M:%S
    path "/mnt/log/salt/minion"
    exclude_path
    pos_file "/mnt/pos/ggcp-salt.pos"
    tag "salt"
    enable_stat_watcher true
    <parse>
      time_format %Y-%m-%d %H:%M:%S
      @type regexp
      expression ^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$
    </parse>
  </source>
  <filter salt.**>
    @type kubernetes_sumologic
    source_category "salt"
    source_name "k8s_salt"
    source_category_prefix "kubernetes/"
    add_stream true
    add_time true
    exclude_namespace_regex "/^(kube-system|monitoring)$/"
  </filter>
  <source>
    @type tail
    format syslog
    path "/mnt/log/startupscript.log"
    exclude_path
    pos_file "/mnt/pos/ggcp-startupscript.log.pos"
    tag "startupscript"
    enable_stat_watcher true
    <parse>
      @type syslog
    </parse>
  </source>
  <filter startupscript.**>
    @type kubernetes_sumologic
    source_category "startupscript"
    source_name "k8s_startupscript"
    source_category_prefix "kubernetes/"
    add_stream true
    add_time true
    exclude_namespace_regex "/^(kube-system|monitoring)$/"
  </filter>
  <source>
    @type tail
    format multiline
    multiline_flush_interval 5s
    format_firstline /^\w\d{4}/
    format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
    time_format %m%d %H:%M:%S.%N
    path "/mnt/log/kubelet.log"
    exclude_path
    pos_file "/mnt/pos/ggcp-kubelet.log.pos"
    tag "kubelet"
    enable_stat_watcher true
    <parse>
      time_format %m%d %H:%M:%S.%N
      format_firstline /^\w\d{4}/
      @type multiline
      format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
    </parse>
  </source>
  <filter kubelet.**>
    @type kubernetes_sumologic
    source_category "kubelet"
    source_name "k8s_kubelet"
    source_category_prefix "kubernetes/"
    add_stream true
    add_time true
    exclude_namespace_regex "/^(kube-system|monitoring)$/"
  </filter>
  <source>
    @type tail
    format json
    time_key timestamp
    time_format %Y-%m-%dT%H:%M:%SZ
    path "/mnt/log/kube-apiserver-audit.log"
    exclude_path
    pos_file "/mnt/pos/ggcp-kube-audit.log.pos"
    tag "kube-audit"
    read_from_head true
    enable_stat_watcher true
    <parse>
      time_key timestamp
      time_format %Y-%m-%dT%H:%M:%SZ
      @type json
      time_type string
    </parse>
  </source>
  <filter kube-audit.**>
    @type kubernetes_sumologic
    source_category "kube-audit"
    source_name "k8s_kube-audit"
    source_category_prefix "kubernetes/"
    add_stream true
    add_time true
    exclude_namespace_regex "/^(kube-system|monitoring)$/"
  </filter>
  <source>
    @type tail
    format multiline
    multiline_flush_interval 5s
    format_firstline /^\w\d{4}/
    format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
    time_format %m%d %H:%M:%S.%N
    path "/mnt/log/glbc.log"
    exclude_path
    pos_file "/mnt/pos/ggcp-glbc.log.pos"
    tag "glbc"
    enable_stat_watcher true
    <parse>
      time_format %m%d %H:%M:%S.%N
      format_firstline /^\w\d{4}/
      @type multiline
      format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
    </parse>
  </source>
  <filter glbc.**>
    @type kubernetes_sumologic
    source_category "glbc"
    source_name "k8s_glbc"
    source_category_prefix "kubernetes/"
    add_stream true
    add_time true
    exclude_namespace_regex "/^(kube-system|monitoring)$/"
  </filter>
  <source>
    @type tail
    format multiline
    multiline_flush_interval 5s
    format_firstline /^\w\d{4}/
    format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
    time_format %m%d %H:%M:%S.%N
    path "/mnt/log/cluster-autoscaler.log"
    exclude_path
    pos_file "/mnt/pos/ggcp-cluster-autoscaler.log.pos"
    tag "cluster-autoscaler"
    enable_stat_watcher true
    <parse>
      time_format %m%d %H:%M:%S.%N
      format_firstline /^\w\d{4}/
      @type multiline
      format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
    </parse>
  </source>
  <filter cluster-autoscaler.**>
    @type kubernetes_sumologic
    source_category "cluster-autoscaler"
    source_name "k8s_cluster-autoscaler"
    source_category_prefix "kubernetes/"
    add_stream true
    add_time true
    exclude_namespace_regex "/^(kube-system|monitoring)$/"
  </filter>
  <match **>
    @type sumologic
    log_key "log"
    endpoint xxxxxx
    verify_ssl true
    log_format "text"
    flush_interval 60
    num_threads 1
    open_timeout 60
    add_timestamp true
    timestamp_key "timestamp"
    proxy_uri ""
    <buffer>
      flush_thread_count 1
      flush_interval 60
    </buffer>
  </match>
</ROOT>