Feature: add Logstash-Forwarder as agent

logstash/indexer/pipeline/kafka_elasticsearch.conf

@@ -2,7 +2,7 @@ input {
 	kafka {
         codec =>  json{}
         bootstrap_servers => "kafka:9092"
-        topics => ["nginx-access", "random", "apache"]
+        topics => ["random", "apache", "random-forwarder", "apache-forwarder"]
         client_id => "logstash_indexer_1"
 	}
 }
@@ -61,6 +61,36 @@ filter {
 			remove_field => [ "timestamp" ]
 		}
 	}
+	if [type] == "random-forwarder" {
+		grok {
+			match => [ "message" , "(?<timestamp>%{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}[- ]%{TIME}) %{NUMBER:pid} %{GREEDYDATA:filename} %{NUMBER:line} %{GREEDYDATA:logger} %{LOGLEVEL:severity} %{GREEDYDATA:quote}"]
+			overwrite => [ "message" ]
+		}
+		date {
+			match => [ "timestamp", "YYYY-MM-dd HH:mm:ss,SSS"]
+			remove_field => [ "timestamp" ]
+		}
+	}
+	if [type] == "apache-forwarder" {
+		grok {
+			match => [ "message" , "%{COMBINEDAPACHELOG}"]
+			overwrite => [ "message" ]
+		}
+		mutate {
+			convert => ["response", "integer"]
+			convert => ["bytes", "integer"]
+			convert => ["responsetime", "float"]
+		}
+		geoip {
+			source => "clientip"
+			target => "geoip"
+			add_tag => [ "apache-geoip" ]
+		}
+		date {
+			match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
+			remove_field => [ "timestamp" ]
+		}
+	}
 }
 
 output {
@@ -69,24 +99,33 @@ output {
 			hosts => ["elasticsearch:9200"]
 			index => "nginx-%{+YYYYMM}"
         }
-        stdout {
-          codec => rubydebug
-        }
 	}
 	if [type] == "random" {
         elasticsearch {
 			hosts => ["elasticsearch:9200"]
 			index => "random-%{+YYYYMM}"
         }
-        stdout {
-          codec => rubydebug
-        }
 	}
 	if [type] == "apache" {
         elasticsearch {
 			hosts => ["elasticsearch:9200"]
 			index => "apache-%{+YYYYMM}"
         }
+	}
+	if [type] == "random-forwarder" {
+        elasticsearch {
+			hosts => ["elasticsearch:9200"]
+			index => "randomforwarder-%{+YYYYMM}"
+        }
+        stdout {
+          codec => rubydebug
+        }
+	}
+	if [type] == "apache-forwarder" {
+        elasticsearch {
+			hosts => ["elasticsearch:9200"]
+			index => "apacheforwarder-%{+YYYYMM}"
+        }
         stdout {
           codec => rubydebug
         }