Febbweiss/docker-log-centralizer
1
0
Fork 0
mirror of https://github.com/Febbweiss/docker-log-centralizer.git synced 2026-03-04 14:25:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feature: add Logstash-Forwarder as agent

Browse Source
This commit is contained in:
ECAILLE Fabrice (externe)
2017-03-17 15:35:36 +01:00
parent e0b11ef1a2
commit 4cc77930d9
8 changed files with 174 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
docker-compose.yml
View File

@@ -26,6 +26,15 @@ services:
- ./logs/apache:/var/log/apache
links:
- shipper
forwarder:
image: apopelo/logstash-forwarder
volumes:
- ./logstash-forwarder/config:/etc/logstash-forwarder
- ./logstash-forwarder/ssl:/etc/ssl
- ./logs/random:/var/log/random
- ./logs/apache:/var/log/apache
links:
- shipper
####################
# Logstash shipper #
####################
@@ -38,6 +47,7 @@ services:
volumes:
- ./logstash/logstash.yml:/usr/share/logstash/config/logstash.yml
- ./logstash/shipper/pipeline/:/usr/share/logstash/pipeline/
- ./logstash/shipper/ssl:/ssl
########################
# Kafka infrastructure #
########################

17
logstash-forwarder/config/config.json Normal file
View File

@@ -0,0 +1,17 @@
{
"network": {
"servers": [ "shipper:5043" ],
"ssl key": "/etc/ssl/selfsigned.key",
"ssl ca": "/etc/ssl/selfsigned.crt"
},
"files": [
{
"paths": [ "/var/log/random/*.log" ],
"fields": { "type": "random-forwarder" }
},
{
"paths": [ "/var/log/apache/*.log" ],
"fields": { "type": "apache-forwarder" }
}
]
}

19
logstash-forwarder/ssl/selfsigned.crt Normal file
View File

@@ -0,0 +1,19 @@
-----BEGIN CERTIFICATE-----
MIIDEjCCAfqgAwIBAgIQPD0a5WB48xwmuFt+FoT1KDANBgkqhkiG9w0BAQsFADAo
MRQwEgYDVQQKEwtMb2cgQ291cmllcjEQMA4GA1UEAxMHc2hpcHBlcjAeFw0xNzAz
MTcxNDE1NDNaFw0yNzAzMTUxNDE1NDNaMCgxFDASBgNVBAoTC0xvZyBDb3VyaWVy
MRAwDgYDVQQDEwdzaGlwcGVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
AQEA5lbWxKD/8lYzglubCldiG95jTSOXFbFc11BpH0OkDsoy6uZWyklfBCGkwTJn
8uZRK0RVV02I7ndhzNm9pVD/1wAClOtNnRs3doUuwjdeQCX/tybujUyQWz63irzb
fWBB70XxgknddKWy4fP4Xu54wNLoaWvTTwRxejTWmrvEDz7PpqlWz7mhBXsxFy09
W5aQpaeBlFR5Pdg+C7yXuTL2oAENQWVMgHpJZqaYB7Vll/JWaHzbbH8kSer9/qIN
Qx9+RWO1SxtuKcpu40NdavT0Km1ZYc3p8i3OOZAJ4pfcB7g7RvUdROElKiO4hWou
Wty046pNWYhum49+zM8r0Q9fSQIDAQABozgwNjAOBgNVHQ8BAf8EBAMCAqQwEwYD
VR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsF
AAOCAQEAO3KmeN/KOFm4OCPKmd5xbZ5I6d/A4V+5lscrFG9tqmF+Ax/hA1RUrnSv
DPk4YSv8zW8tGrZpX3iLGGX7giqT6EcKH+leyyU0pvO2oUvs+R0cHu4kY7XknUHm
Lx3QovDap4+uZIZdawI23fGyKSa7PUybKc1fuxrwZNns3zVc+Lp+iSAfsPWI43kX
ZR0WlMzXIr8EVef+Hz2xLdDsuiwUVrBSWH+tn3pedwPjeEO/YgH+CV899hWlDBl+
1r8+S11s+Ur+IMuydygLds8QDgLJlFmJVZmvifs+fEd6jPu34CsvA6tF0YqW03iw
Nmnzx5bMkV9Iffg/rdpYapPIV877DQ==
-----END CERTIFICATE-----

27
logstash-forwarder/ssl/selfsigned.key Normal file
View File

@@ -0,0 +1,27 @@
-----BEGIN RSA PRIVATE KEY-----
MIIEowIBAAKCAQEA5lbWxKD/8lYzglubCldiG95jTSOXFbFc11BpH0OkDsoy6uZW
yklfBCGkwTJn8uZRK0RVV02I7ndhzNm9pVD/1wAClOtNnRs3doUuwjdeQCX/tybu
jUyQWz63irzbfWBB70XxgknddKWy4fP4Xu54wNLoaWvTTwRxejTWmrvEDz7PpqlW
z7mhBXsxFy09W5aQpaeBlFR5Pdg+C7yXuTL2oAENQWVMgHpJZqaYB7Vll/JWaHzb
bH8kSer9/qINQx9+RWO1SxtuKcpu40NdavT0Km1ZYc3p8i3OOZAJ4pfcB7g7RvUd
ROElKiO4hWouWty046pNWYhum49+zM8r0Q9fSQIDAQABAoIBAF3BsyHONuA5XjYN
e4o2D2UwnFLlzbWywAuUW3WyXrD2dMtQfrtQKjDQUPOixNUGErv90I78rGQEi/aK
GruTwM+O3X4oWSs9zNGdXG8Jgn+x9+hhHHC/XXMGkEcUoHLkH3J5kdhqoI2+dJWs
bMiXYOd78cYXcaeUM2x+3WuWSjO5o6zNDka4RnwFbVk2k2Zay1EsSzjb+9g78Y+C
V+bWYz/o5JdWue5ZmnlvWLS/l8YmlYFaIiHePcbQUtiLDGpIB9LYn63lijRIsDdJ
S+zkxLJfnE9h8b3eViv2h32Ysxd0HCz8on3U6WpvtJ1zTSLZM/wps+clfvHAQxUS
ad1DJBECgYEA9y3cbh62TX9YuDwVNXSpuszAuqSHxXvVo1AQMux8GQnOGyuBwwww
0Nj6nAUOLdIIPhs5fY88D1UJa6TH3s4lnRFURrYa5ueeE0DRU9fVDJ5nnwcPlaXg
bLVRtW7oP3xnFYsB3MQtMANQoHXxqHeO0ttwUI7RhydSY8ctLoZ0cI8CgYEA7o8i
Z5l+9TSQZefqa/eGhOPsEhYnyWfha9R42KOUkPvEu6awgk3UAAdgzY/MN636Gv1O
shWcJGC6K8/pQhkc3jhRICJUZutkUh1r5zKdOKp/lFyGqHF1r7nwhouPg4PDMot1
z0bv2Q8XRxKoXcDZWj7pQog3LAUKWbub7z677qcCgYEAycm40HVmInI5/X2fWtp4
zbTmCfjTllb4G7D66HfC/7XPesLT1mWXXIgmX0SJRMLYU8tp7aUHRQUJAcWuHahx
cH57LJbx81tW4yThPu9OUFTpBxx+jo6yXkzP/awjEeD/TskCpjXJJg8uvTBMLloc
UOswD7PoPcvOKQKrUfYWDOUCgYA0rOhU5eDRtfyYc+AgXPSYDVfIGppIb6anz6+r
bltUtDMZmFHrQ9Im+3oVicX3GXRbV0l0Ky9iHPWezPbdgHwOWXMYXFw0qY9qTlLQ
Jy3uID+xe8jEEuGAntcMbm6PBJ4qX/7weOJnFFz4dZc1tGqJ8dBBv6AnGSnS7Ikf
B8QpiwKBgDW+HqJQTwEZMGLHLq3GNlgQqzDQrSRhGMyueeUK1VYPNRpAQQO/KZy/
7BjjjJswdoF3gZqGBciKYGfqRvojyNOijadJnicuoVNcLqxtZCB4PbFLUfpz00fM
9AQkK0ejTa2/Xh0hF19SqdaK5iiEmFbrDNxW/EastBquOYPAiC1m
-----END RSA PRIVATE KEY-----

53
logstash/indexer/pipeline/kafka_elasticsearch.conf
View File

@@ -2,7 +2,7 @@ input {
kafka {
codec => json{}
bootstrap_servers => "kafka:9092"
topics => ["nginx-access", "random", "apache"]
topics => ["random", "apache", "random-forwarder", "apache-forwarder"]
client_id => "logstash_indexer_1"
}
}
@@ -61,6 +61,36 @@ filter {
remove_field => [ "timestamp" ]
}
}
if [type] == "random-forwarder" {
grok {
match => [ "message" , "(?<timestamp>%{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}[- ]%{TIME}) %{NUMBER:pid} %{GREEDYDATA:filename} %{NUMBER:line} %{GREEDYDATA:logger} %{LOGLEVEL:severity} %{GREEDYDATA:quote}"]
overwrite => [ "message" ]
}
date {
match => [ "timestamp", "YYYY-MM-dd HH:mm:ss,SSS"]
remove_field => [ "timestamp" ]
}
}
if [type] == "apache-forwarder" {
grok {
match => [ "message" , "%{COMBINEDAPACHELOG}"]
overwrite => [ "message" ]
}
mutate {
convert => ["response", "integer"]
convert => ["bytes", "integer"]
convert => ["responsetime", "float"]
}
geoip {
source => "clientip"
target => "geoip"
add_tag => [ "apache-geoip" ]
}
date {
match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
remove_field => [ "timestamp" ]
}
}
}
output {
@@ -69,24 +99,33 @@ output {
hosts => ["elasticsearch:9200"]
index => "nginx-%{+YYYYMM}"
}
stdout {
codec => rubydebug
}
}
if [type] == "random" {
elasticsearch {
hosts => ["elasticsearch:9200"]
index => "random-%{+YYYYMM}"
}
stdout {
codec => rubydebug
}
}
if [type] == "apache" {
elasticsearch {
hosts => ["elasticsearch:9200"]
index => "apache-%{+YYYYMM}"
}
}
if [type] == "random-forwarder" {
elasticsearch {
hosts => ["elasticsearch:9200"]
index => "randomforwarder-%{+YYYYMM}"
}
stdout {
codec => rubydebug
}
}
if [type] == "apache-forwarder" {
elasticsearch {
hosts => ["elasticsearch:9200"]
index => "apacheforwarder-%{+YYYYMM}"
}
stdout {
codec => rubydebug
}

11
logstash/shipper/pipeline/beat_kafka.conf
View File

@@ -2,6 +2,11 @@ input {
beats {
port => 5044
}
lumberjack {
port => 5043
ssl_key => "/ssl/selfsigned.key"
ssl_certificate => "/ssl/selfsigned.crt"
}
}
output {
kafka {
@@ -9,7 +14,9 @@ output {
bootstrap_servers => "kafka:9092"
topic_id => "%{type}"
}
stdout {
codec => rubydebug
if [type] == "apache-forwarder" || [type] == "random-forwarder" {
stdout {
codec => rubydebug
}
}
}

19
logstash/shipper/ssl/selfsigned.crt Normal file
View File

@@ -0,0 +1,19 @@
-----BEGIN CERTIFICATE-----
MIIDEjCCAfqgAwIBAgIQPD0a5WB48xwmuFt+FoT1KDANBgkqhkiG9w0BAQsFADAo
MRQwEgYDVQQKEwtMb2cgQ291cmllcjEQMA4GA1UEAxMHc2hpcHBlcjAeFw0xNzAz
MTcxNDE1NDNaFw0yNzAzMTUxNDE1NDNaMCgxFDASBgNVBAoTC0xvZyBDb3VyaWVy
MRAwDgYDVQQDEwdzaGlwcGVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
AQEA5lbWxKD/8lYzglubCldiG95jTSOXFbFc11BpH0OkDsoy6uZWyklfBCGkwTJn
8uZRK0RVV02I7ndhzNm9pVD/1wAClOtNnRs3doUuwjdeQCX/tybujUyQWz63irzb
fWBB70XxgknddKWy4fP4Xu54wNLoaWvTTwRxejTWmrvEDz7PpqlWz7mhBXsxFy09
W5aQpaeBlFR5Pdg+C7yXuTL2oAENQWVMgHpJZqaYB7Vll/JWaHzbbH8kSer9/qIN
Qx9+RWO1SxtuKcpu40NdavT0Km1ZYc3p8i3OOZAJ4pfcB7g7RvUdROElKiO4hWou
Wty046pNWYhum49+zM8r0Q9fSQIDAQABozgwNjAOBgNVHQ8BAf8EBAMCAqQwEwYD
VR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsF
AAOCAQEAO3KmeN/KOFm4OCPKmd5xbZ5I6d/A4V+5lscrFG9tqmF+Ax/hA1RUrnSv
DPk4YSv8zW8tGrZpX3iLGGX7giqT6EcKH+leyyU0pvO2oUvs+R0cHu4kY7XknUHm
Lx3QovDap4+uZIZdawI23fGyKSa7PUybKc1fuxrwZNns3zVc+Lp+iSAfsPWI43kX
ZR0WlMzXIr8EVef+Hz2xLdDsuiwUVrBSWH+tn3pedwPjeEO/YgH+CV899hWlDBl+
1r8+S11s+Ur+IMuydygLds8QDgLJlFmJVZmvifs+fEd6jPu34CsvA6tF0YqW03iw
Nmnzx5bMkV9Iffg/rdpYapPIV877DQ==
-----END CERTIFICATE-----

27
logstash/shipper/ssl/selfsigned.key Normal file
View File

@@ -0,0 +1,27 @@
-----BEGIN RSA PRIVATE KEY-----
MIIEowIBAAKCAQEA5lbWxKD/8lYzglubCldiG95jTSOXFbFc11BpH0OkDsoy6uZW
yklfBCGkwTJn8uZRK0RVV02I7ndhzNm9pVD/1wAClOtNnRs3doUuwjdeQCX/tybu
jUyQWz63irzbfWBB70XxgknddKWy4fP4Xu54wNLoaWvTTwRxejTWmrvEDz7PpqlW
z7mhBXsxFy09W5aQpaeBlFR5Pdg+C7yXuTL2oAENQWVMgHpJZqaYB7Vll/JWaHzb
bH8kSer9/qINQx9+RWO1SxtuKcpu40NdavT0Km1ZYc3p8i3OOZAJ4pfcB7g7RvUd
ROElKiO4hWouWty046pNWYhum49+zM8r0Q9fSQIDAQABAoIBAF3BsyHONuA5XjYN
e4o2D2UwnFLlzbWywAuUW3WyXrD2dMtQfrtQKjDQUPOixNUGErv90I78rGQEi/aK
GruTwM+O3X4oWSs9zNGdXG8Jgn+x9+hhHHC/XXMGkEcUoHLkH3J5kdhqoI2+dJWs
bMiXYOd78cYXcaeUM2x+3WuWSjO5o6zNDka4RnwFbVk2k2Zay1EsSzjb+9g78Y+C
V+bWYz/o5JdWue5ZmnlvWLS/l8YmlYFaIiHePcbQUtiLDGpIB9LYn63lijRIsDdJ
S+zkxLJfnE9h8b3eViv2h32Ysxd0HCz8on3U6WpvtJ1zTSLZM/wps+clfvHAQxUS
ad1DJBECgYEA9y3cbh62TX9YuDwVNXSpuszAuqSHxXvVo1AQMux8GQnOGyuBwwww
0Nj6nAUOLdIIPhs5fY88D1UJa6TH3s4lnRFURrYa5ueeE0DRU9fVDJ5nnwcPlaXg
bLVRtW7oP3xnFYsB3MQtMANQoHXxqHeO0ttwUI7RhydSY8ctLoZ0cI8CgYEA7o8i
Z5l+9TSQZefqa/eGhOPsEhYnyWfha9R42KOUkPvEu6awgk3UAAdgzY/MN636Gv1O
shWcJGC6K8/pQhkc3jhRICJUZutkUh1r5zKdOKp/lFyGqHF1r7nwhouPg4PDMot1
z0bv2Q8XRxKoXcDZWj7pQog3LAUKWbub7z677qcCgYEAycm40HVmInI5/X2fWtp4
zbTmCfjTllb4G7D66HfC/7XPesLT1mWXXIgmX0SJRMLYU8tp7aUHRQUJAcWuHahx
cH57LJbx81tW4yThPu9OUFTpBxx+jo6yXkzP/awjEeD/TskCpjXJJg8uvTBMLloc
UOswD7PoPcvOKQKrUfYWDOUCgYA0rOhU5eDRtfyYc+AgXPSYDVfIGppIb6anz6+r
bltUtDMZmFHrQ9Im+3oVicX3GXRbV0l0Ky9iHPWezPbdgHwOWXMYXFw0qY9qTlLQ
Jy3uID+xe8jEEuGAntcMbm6PBJ4qX/7weOJnFFz4dZc1tGqJ8dBBv6AnGSnS7Ikf
B8QpiwKBgDW+HqJQTwEZMGLHLq3GNlgQqzDQrSRhGMyueeUK1VYPNRpAQQO/KZy/
7BjjjJswdoF3gZqGBciKYGfqRvojyNOijadJnicuoVNcLqxtZCB4PbFLUfpz00fM
9AQkK0ejTa2/Xh0hF19SqdaK5iiEmFbrDNxW/EastBquOYPAiC1m
-----END RSA PRIVATE KEY-----