diff --git a/docker-compose.yml b/docker-compose.yml index 564b045..61e8ce2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,10 @@ services: image: febbweiss/apache-log-generator volumes: - ./logs/apache:/var/log/apache + java_log_generator: + image: febbweiss/java-log-generator + volumes: + - ./logs/java:/var/log/java random_log_generator: # Star Wars quote generator image: davidmccormick/random_log_generator command: python log_generator.py --logFile /var/log/random/random.log @@ -33,6 +37,14 @@ services: - ./logs/apache:/var/log/apache links: - shipper + rsyslog: + image: camptocamp/rsyslog-bin + volumes: + - ./rsyslog/conf.d:/etc/rsyslog-confd + - ./rsyslog/rsyslog.conf:/etc/rsyslog.conf + - ./logs/java:/var/log/java + links: + - shipper #################### # Logstash shipper # #################### diff --git a/logstash/indexer/pipeline/kafka_elasticsearch.conf b/logstash/indexer/pipeline/kafka_elasticsearch.conf index 3b74dce..4fa34d3 100644 --- a/logstash/indexer/pipeline/kafka_elasticsearch.conf +++ b/logstash/indexer/pipeline/kafka_elasticsearch.conf @@ -23,6 +23,12 @@ input { topics => ["apache-forwarder"] client_id => "logstash_indexer_1" } + kafka { + codec => json{} + bootstrap_servers => "kafka:9092" + topics => ["javalog"] + client_id => "logstash_indexer_1" + } } filter { @@ -130,6 +136,12 @@ output { index => "apache-%{+YYYYMM}" } } + if [type] == "javalog" { + elasticsearch { + hosts => ["elasticsearch:9200"] + index => "javalog-%{+YYYYMM}" + } + } if [type] == "random-forwarder" { elasticsearch { hosts => ["elasticsearch:9200"] diff --git a/logstash/shipper/pipeline/beat_kafka.conf b/logstash/shipper/pipeline/beat_kafka.conf index fca44a5..1a11d78 100644 --- a/logstash/shipper/pipeline/beat_kafka.conf +++ b/logstash/shipper/pipeline/beat_kafka.conf @@ -2,12 +2,33 @@ input { beats { port => 5044 } + udp { + port => 10514 + type => "syslog" + } lumberjack { port => 5043 ssl_key => "/ssl/selfsigned.key" ssl_certificate => "/ssl/selfsigned.crt" } } + +filter { + if [type] == "syslog" { + mutate { + gsub => [ "message", "\t", "\\t" ] + } + if ![programname] { + json { + source => "message" + } + } + mutate { + replace => [ "type", "%{programname}" ] + } + } +} + output { kafka { codec => json diff --git a/rsyslog/conf.d/rsyslog-json.conf b/rsyslog/conf.d/rsyslog-json.conf new file mode 100644 index 0000000..74356dc --- /dev/null +++ b/rsyslog/conf.d/rsyslog-json.conf @@ -0,0 +1,11 @@ +template(name="ls_json" + type="list" + option.json="on") { + constant(value="{") + constant(value="\"@timestamp\":\"") property(name="timereported" dateFormat="rfc3339") + constant(value="\",\"message\":\"") property(name="msg") + constant(value="\",\"host\":\"") property(name="hostname") + constant(value="\",\"programname\":\"") property(name="programname") + constant(value="\",\"procid\":\"") property(name="procid") + constant(value="\"}") + } \ No newline at end of file diff --git a/rsyslog/conf.d/rsyslog.conf b/rsyslog/conf.d/rsyslog.conf new file mode 100644 index 0000000..93be18c --- /dev/null +++ b/rsyslog/conf.d/rsyslog.conf @@ -0,0 +1,9 @@ +module(load="imfile" PollingInterval="10" mode="inotify") #needs to be done just once + +input(type="imfile" + File="/var/log/java/*.log" + Tag="javalog" + PersistStateInterval="100" + Severity="info" + startmsg.regex="^[[:digit:]]{1,2}-[[:digit:]]{1,2}-[[:digit:]]{1,4} [[:digit:]]{1,2}:[[:digit:]]{1,2}:[[:digit:]]{1,2}" +) \ No newline at end of file diff --git a/rsyslog/rsyslog.conf b/rsyslog/rsyslog.conf new file mode 100644 index 0000000..f162778 --- /dev/null +++ b/rsyslog/rsyslog.conf @@ -0,0 +1,3 @@ +$IncludeConfig /etc/rsyslog-confd/*.conf + +*.* @shipper:10514;ls_json