Feature: add rsyslog multiline logs

docker-compose.yml

@@ -8,6 +8,10 @@ services:
   image: febbweiss/apache-log-generator
   volumes:
    - ./logs/apache:/var/log/apache
+ java_log_generator:
+  image: febbweiss/java-log-generator
+  volumes:
+   - ./logs/java:/var/log/java
  random_log_generator: # Star Wars quote generator
   image: davidmccormick/random_log_generator
   command: python log_generator.py --logFile /var/log/random/random.log
@@ -33,6 +37,14 @@ services:
    - ./logs/apache:/var/log/apache
   links:
    - shipper
+ rsyslog:
+  image: camptocamp/rsyslog-bin
+  volumes:
+   - ./rsyslog/conf.d:/etc/rsyslog-confd
+   - ./rsyslog/rsyslog.conf:/etc/rsyslog.conf
+   - ./logs/java:/var/log/java
+  links:
+   - shipper
  ####################
  # Logstash shipper #
  ####################

logstash/indexer/pipeline/kafka_elasticsearch.conf

@@ -23,6 +23,12 @@ input {
         topics => ["apache-forwarder"]
         client_id => "logstash_indexer_1"
 	}
+	kafka {
+        codec =>  json{}
+        bootstrap_servers => "kafka:9092"
+        topics => ["javalog"]
+        client_id => "logstash_indexer_1"
+	}
 }
 
 filter {
@@ -130,6 +136,12 @@ output {
 			index => "apache-%{+YYYYMM}"
         }
 	}
+	if [type] == "javalog" {
+        elasticsearch {
+			hosts => ["elasticsearch:9200"]
+			index => "javalog-%{+YYYYMM}"
+        }
+	}
 	if [type] == "random-forwarder" {
         elasticsearch {
 			hosts => ["elasticsearch:9200"]

logstash/shipper/pipeline/beat_kafka.conf

@@ -2,12 +2,33 @@ input {
   beats {
     port => 5044
   }
+  udp {
+    port => 10514
+    type => "syslog"
+  }
   lumberjack {
 	port => 5043
 	ssl_key => "/ssl/selfsigned.key"
 	ssl_certificate => "/ssl/selfsigned.crt"
   }
 }
+
+filter {
+  if [type] == "syslog" {
+    mutate {
+	  gsub => [ "message", "\t", "\\t" ]
+    }
+	if ![programname] {
+	  json {
+	  	source => "message"
+	  }
+	}
+	mutate {
+      replace => [ "type", "%{programname}" ]
+    }
+  }
+}
+
 output {
   kafka {
     codec => json

rsyslog/conf.d/rsyslog-json.conf

@@ -0,0 +1,11 @@
+template(name="ls_json"
+	type="list"
+	option.json="on") {
+		constant(value="{")
+		constant(value="\"@timestamp\":\"")     property(name="timereported" dateFormat="rfc3339")
+		constant(value="\",\"message\":\"")     property(name="msg")
+		constant(value="\",\"host\":\"")        property(name="hostname")
+		constant(value="\",\"programname\":\"") property(name="programname")
+		constant(value="\",\"procid\":\"")      property(name="procid")
+		constant(value="\"}")
+	}

rsyslog/conf.d/rsyslog.conf

@@ -0,0 +1,9 @@
+module(load="imfile" PollingInterval="10" mode="inotify") #needs to be done just once
+
+input(type="imfile"
+  File="/var/log/java/*.log"
+  Tag="javalog"
+  PersistStateInterval="100"
+  Severity="info"
+  startmsg.regex="^[[:digit:]]{1,2}-[[:digit:]]{1,2}-[[:digit:]]{1,4} [[:digit:]]{1,2}:[[:digit:]]{1,2}:[[:digit:]]{1,2}"
+)

rsyslog/rsyslog.conf

@@ -0,0 +1,3 @@
+$IncludeConfig /etc/rsyslog-confd/*.conf
+		 
+*.* @shipper:10514;ls_json