fluentd.yaml

elasticsearch:
  hosts: ['http://elastic-elasticsearch-master:9200']
  logstash_prefix: 'kubernetes_cluster'

configMaps:
  useDefaults:
    containersInputConf: false
    systemInputConf: false

extraConfigMaps:
  containers.input.conf: |-
    <source>
    @id fluentd-containers.log
    @type tail
    path /var/log/containers/*.log
    pos_file /var/log/containers.log.pos
    tag raw.kubernetes.*
    read_from_head true
    <parse>
    @type multi_format
    <pattern>
    format json
    time_key time
    time_format %Y-%m-%dT%H:%M:%S.%NZ
    </pattern>
    <pattern>
    format /^(?<time>.+) (?<stream>stdout|stderr) [^ ]* (?<log>.*)$/
    time_format %Y-%m-%dT%H:%M:%S.%N%:z
    </pattern>
    </parse>
    </source>

    # Detect exceptions in the log output and forward them as one log entry.
    <match raw.kubernetes.**>
    @id raw.kubernetes
    @type detect_exceptions
    remove_tag_prefix raw
    message log
    stream stream
    multiline_flush_interval 5
    max_bytes 500000
    max_lines 1000
    </match>

    # Concatenate multi-line logs
    <filter **>
    @id filter_concat
    @type concat
    key message
    multiline_end_regexp /\n$/
    separator ""
    </filter>

    # Enriches records with Kubernetes metadata
    <filter kubernetes.**>
    @id filter_kubernetes_metadata
    @type kubernetes_metadata
    </filter>

    # Fixes json fields in Elasticsearch
    <filter kubernetes.**>
    @id filter_parser
    @type parser
    key_name log
    reserve_data true
    remove_key_name_field true
    <parse>
    @type multi_format
    <pattern>
    format json
    </pattern>
    <pattern>
    format none
    </pattern>
    </parse>
    </filter>

    #exclude kube-system
    <match kubernetes.var.log.containers.**kube-system**.log>
    @type null
    </match>

    # Filter to only records with label fluentd=true
    <filter kubernetes.**>
    @type grep
    <regexp>
    key $.kubernetes.labels.fluentd
    pattern true
    </regexp>
    </filter>

    <filter kubernetes.**>
    @type grep
    <exclude>
    key $.kubernetes.container_name
    pattern istio-proxy
    </exclude>
    </filter>