shipper.conf

input {
    tcp {
        port => 10514
        mode => "server"
        tags => ['nova', 'oslofmt']
        type => "nova"
    }
    tcp {
        port => 11514
        mode => "server"
        tags => ['cinder', 'oslofmt']
        type => "cinder"
    }
    tcp {
        port => 12514
        mode => "server"
        tags => ['neutron', 'oslofmt']
        type => "neutron"
    }
    tcp {
        port => 13514
        mode => "server"
        tags => ['keystone', 'oslofmt']
        type => "keystone"
    }
    tcp {
        port => 14514
        mode => "server"
        tags => ['glance', 'oslofmt']
        type => "glance"
    }
#    tcp {
#        port => 1514
#        mode => "server"
#        tags => ['syslog']
#        type => "syslog"
#    }
}

filter {
  if "oslofmt" in [tags]{
    grok {
      # Do multiline matching as the above mutliline filter may add newlines
      # to the log messages.
      # TODO move the LOGLEVELs into a proper grok pattern.
      patterns_dir => "/logstash_patterns"
      match => { "message" => "%{OPENSTACK_SYSLOGLINE}" }
      add_field => { "received_at" => "%{@timestamp}" }
      add_field => { "received_from" => "%{host}" }
      add_field => { "_message" => "%{syslog5424_host} %{message}" }
    }
    if ("_grokparsefailure" not in [tags]){
       syslog_pri {
          severity_labels => ["EMERGENCY", "ALERT", "CRITICAL", "NOTICE", "ERROR", "INFO", "DEBUG" ]
          syslog_pri_field_name => "syslog5424_pri"
       }
       mutate {
         rename =>  ["msg", "message"]
         rename => ["syslog5424_host", "host"]
         remove_field => "syslog_ts"
         remove_field => "syslog5424_pri"
         remove_field => "os_message"
         add_tag => ["processed", "openstack_syslog", "filter_10.32.105.107"]
       }
    }
  } else if [type] == "syslog" {
     grok {
#match => { "message" => "%{SYSLOG5424PRI:priority}%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
        match => { "message" => "%{SYSLOG5424PRI:priority}%{TIMESTAMP_ISO8601:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
        add_field => [ "received_at", "%{@timestamp}" ]
        add_field => [ "received_from", "%{host}" ]
     }
    mutate {
      add_tag => ["processed"]
    }
  }
}
output {
  redis {
#data_type => "list"
    data_type => "channel"
	key =>  "logstash-*"
 	host => "broker01"
	port => 6379
  }
}