diff --git a/docker-compose.yml b/docker-compose.yml index b3f2da7..f7c6fb3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,6 +26,15 @@ services: - ./logs/apache:/var/log/apache links: - shipper + forwarder: + image: apopelo/logstash-forwarder + volumes: + - ./logstash-forwarder/config:/etc/logstash-forwarder + - ./logstash-forwarder/ssl:/etc/ssl + - ./logs/random:/var/log/random + - ./logs/apache:/var/log/apache + links: + - shipper #################### # Logstash shipper # #################### @@ -38,6 +47,7 @@ services: volumes: - ./logstash/logstash.yml:/usr/share/logstash/config/logstash.yml - ./logstash/shipper/pipeline/:/usr/share/logstash/pipeline/ + - ./logstash/shipper/ssl:/ssl ######################## # Kafka infrastructure # ######################## diff --git a/logstash-forwarder/config/config.json b/logstash-forwarder/config/config.json new file mode 100644 index 0000000..ee529dd --- /dev/null +++ b/logstash-forwarder/config/config.json @@ -0,0 +1,17 @@ +{ + "network": { + "servers": [ "shipper:5043" ], + "ssl key": "/etc/ssl/selfsigned.key", + "ssl ca": "/etc/ssl/selfsigned.crt" + }, + "files": [ + { + "paths": [ "/var/log/random/*.log" ], + "fields": { "type": "random-forwarder" } + }, + { + "paths": [ "/var/log/apache/*.log" ], + "fields": { "type": "apache-forwarder" } + } + ] +} \ No newline at end of file diff --git a/logstash-forwarder/ssl/selfsigned.crt b/logstash-forwarder/ssl/selfsigned.crt new file mode 100644 index 0000000..216b191 --- /dev/null +++ b/logstash-forwarder/ssl/selfsigned.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDEjCCAfqgAwIBAgIQPD0a5WB48xwmuFt+FoT1KDANBgkqhkiG9w0BAQsFADAo +MRQwEgYDVQQKEwtMb2cgQ291cmllcjEQMA4GA1UEAxMHc2hpcHBlcjAeFw0xNzAz +MTcxNDE1NDNaFw0yNzAzMTUxNDE1NDNaMCgxFDASBgNVBAoTC0xvZyBDb3VyaWVy +MRAwDgYDVQQDEwdzaGlwcGVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC +AQEA5lbWxKD/8lYzglubCldiG95jTSOXFbFc11BpH0OkDsoy6uZWyklfBCGkwTJn +8uZRK0RVV02I7ndhzNm9pVD/1wAClOtNnRs3doUuwjdeQCX/tybujUyQWz63irzb +fWBB70XxgknddKWy4fP4Xu54wNLoaWvTTwRxejTWmrvEDz7PpqlWz7mhBXsxFy09 +W5aQpaeBlFR5Pdg+C7yXuTL2oAENQWVMgHpJZqaYB7Vll/JWaHzbbH8kSer9/qIN +Qx9+RWO1SxtuKcpu40NdavT0Km1ZYc3p8i3OOZAJ4pfcB7g7RvUdROElKiO4hWou +Wty046pNWYhum49+zM8r0Q9fSQIDAQABozgwNjAOBgNVHQ8BAf8EBAMCAqQwEwYD +VR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsF +AAOCAQEAO3KmeN/KOFm4OCPKmd5xbZ5I6d/A4V+5lscrFG9tqmF+Ax/hA1RUrnSv +DPk4YSv8zW8tGrZpX3iLGGX7giqT6EcKH+leyyU0pvO2oUvs+R0cHu4kY7XknUHm +Lx3QovDap4+uZIZdawI23fGyKSa7PUybKc1fuxrwZNns3zVc+Lp+iSAfsPWI43kX +ZR0WlMzXIr8EVef+Hz2xLdDsuiwUVrBSWH+tn3pedwPjeEO/YgH+CV899hWlDBl+ +1r8+S11s+Ur+IMuydygLds8QDgLJlFmJVZmvifs+fEd6jPu34CsvA6tF0YqW03iw +Nmnzx5bMkV9Iffg/rdpYapPIV877DQ== +-----END CERTIFICATE----- diff --git a/logstash-forwarder/ssl/selfsigned.key b/logstash-forwarder/ssl/selfsigned.key new file mode 100644 index 0000000..4d00c20 --- /dev/null +++ b/logstash-forwarder/ssl/selfsigned.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA5lbWxKD/8lYzglubCldiG95jTSOXFbFc11BpH0OkDsoy6uZW +yklfBCGkwTJn8uZRK0RVV02I7ndhzNm9pVD/1wAClOtNnRs3doUuwjdeQCX/tybu +jUyQWz63irzbfWBB70XxgknddKWy4fP4Xu54wNLoaWvTTwRxejTWmrvEDz7PpqlW +z7mhBXsxFy09W5aQpaeBlFR5Pdg+C7yXuTL2oAENQWVMgHpJZqaYB7Vll/JWaHzb +bH8kSer9/qINQx9+RWO1SxtuKcpu40NdavT0Km1ZYc3p8i3OOZAJ4pfcB7g7RvUd +ROElKiO4hWouWty046pNWYhum49+zM8r0Q9fSQIDAQABAoIBAF3BsyHONuA5XjYN +e4o2D2UwnFLlzbWywAuUW3WyXrD2dMtQfrtQKjDQUPOixNUGErv90I78rGQEi/aK +GruTwM+O3X4oWSs9zNGdXG8Jgn+x9+hhHHC/XXMGkEcUoHLkH3J5kdhqoI2+dJWs +bMiXYOd78cYXcaeUM2x+3WuWSjO5o6zNDka4RnwFbVk2k2Zay1EsSzjb+9g78Y+C +V+bWYz/o5JdWue5ZmnlvWLS/l8YmlYFaIiHePcbQUtiLDGpIB9LYn63lijRIsDdJ +S+zkxLJfnE9h8b3eViv2h32Ysxd0HCz8on3U6WpvtJ1zTSLZM/wps+clfvHAQxUS +ad1DJBECgYEA9y3cbh62TX9YuDwVNXSpuszAuqSHxXvVo1AQMux8GQnOGyuBwwww +0Nj6nAUOLdIIPhs5fY88D1UJa6TH3s4lnRFURrYa5ueeE0DRU9fVDJ5nnwcPlaXg +bLVRtW7oP3xnFYsB3MQtMANQoHXxqHeO0ttwUI7RhydSY8ctLoZ0cI8CgYEA7o8i +Z5l+9TSQZefqa/eGhOPsEhYnyWfha9R42KOUkPvEu6awgk3UAAdgzY/MN636Gv1O +shWcJGC6K8/pQhkc3jhRICJUZutkUh1r5zKdOKp/lFyGqHF1r7nwhouPg4PDMot1 +z0bv2Q8XRxKoXcDZWj7pQog3LAUKWbub7z677qcCgYEAycm40HVmInI5/X2fWtp4 +zbTmCfjTllb4G7D66HfC/7XPesLT1mWXXIgmX0SJRMLYU8tp7aUHRQUJAcWuHahx +cH57LJbx81tW4yThPu9OUFTpBxx+jo6yXkzP/awjEeD/TskCpjXJJg8uvTBMLloc +UOswD7PoPcvOKQKrUfYWDOUCgYA0rOhU5eDRtfyYc+AgXPSYDVfIGppIb6anz6+r +bltUtDMZmFHrQ9Im+3oVicX3GXRbV0l0Ky9iHPWezPbdgHwOWXMYXFw0qY9qTlLQ +Jy3uID+xe8jEEuGAntcMbm6PBJ4qX/7weOJnFFz4dZc1tGqJ8dBBv6AnGSnS7Ikf +B8QpiwKBgDW+HqJQTwEZMGLHLq3GNlgQqzDQrSRhGMyueeUK1VYPNRpAQQO/KZy/ +7BjjjJswdoF3gZqGBciKYGfqRvojyNOijadJnicuoVNcLqxtZCB4PbFLUfpz00fM +9AQkK0ejTa2/Xh0hF19SqdaK5iiEmFbrDNxW/EastBquOYPAiC1m +-----END RSA PRIVATE KEY----- diff --git a/logstash/indexer/pipeline/kafka_elasticsearch.conf b/logstash/indexer/pipeline/kafka_elasticsearch.conf index 551c687..7b18fd7 100644 --- a/logstash/indexer/pipeline/kafka_elasticsearch.conf +++ b/logstash/indexer/pipeline/kafka_elasticsearch.conf @@ -2,7 +2,7 @@ input { kafka { codec => json{} bootstrap_servers => "kafka:9092" - topics => ["nginx-access", "random", "apache"] + topics => ["random", "apache", "random-forwarder", "apache-forwarder"] client_id => "logstash_indexer_1" } } @@ -61,6 +61,36 @@ filter { remove_field => [ "timestamp" ] } } + if [type] == "random-forwarder" { + grok { + match => [ "message" , "(?%{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}[- ]%{TIME}) %{NUMBER:pid} %{GREEDYDATA:filename} %{NUMBER:line} %{GREEDYDATA:logger} %{LOGLEVEL:severity} %{GREEDYDATA:quote}"] + overwrite => [ "message" ] + } + date { + match => [ "timestamp", "YYYY-MM-dd HH:mm:ss,SSS"] + remove_field => [ "timestamp" ] + } + } + if [type] == "apache-forwarder" { + grok { + match => [ "message" , "%{COMBINEDAPACHELOG}"] + overwrite => [ "message" ] + } + mutate { + convert => ["response", "integer"] + convert => ["bytes", "integer"] + convert => ["responsetime", "float"] + } + geoip { + source => "clientip" + target => "geoip" + add_tag => [ "apache-geoip" ] + } + date { + match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ] + remove_field => [ "timestamp" ] + } + } } output { @@ -69,24 +99,33 @@ output { hosts => ["elasticsearch:9200"] index => "nginx-%{+YYYYMM}" } - stdout { - codec => rubydebug - } } if [type] == "random" { elasticsearch { hosts => ["elasticsearch:9200"] index => "random-%{+YYYYMM}" } - stdout { - codec => rubydebug - } } if [type] == "apache" { elasticsearch { hosts => ["elasticsearch:9200"] index => "apache-%{+YYYYMM}" } + } + if [type] == "random-forwarder" { + elasticsearch { + hosts => ["elasticsearch:9200"] + index => "randomforwarder-%{+YYYYMM}" + } + stdout { + codec => rubydebug + } + } + if [type] == "apache-forwarder" { + elasticsearch { + hosts => ["elasticsearch:9200"] + index => "apacheforwarder-%{+YYYYMM}" + } stdout { codec => rubydebug } diff --git a/logstash/shipper/pipeline/beat_kafka.conf b/logstash/shipper/pipeline/beat_kafka.conf index f43d743..9b2ebc6 100644 --- a/logstash/shipper/pipeline/beat_kafka.conf +++ b/logstash/shipper/pipeline/beat_kafka.conf @@ -2,6 +2,11 @@ input { beats { port => 5044 } + lumberjack { + port => 5043 + ssl_key => "/ssl/selfsigned.key" + ssl_certificate => "/ssl/selfsigned.crt" + } } output { kafka { @@ -9,7 +14,9 @@ output { bootstrap_servers => "kafka:9092" topic_id => "%{type}" } - stdout { - codec => rubydebug + if [type] == "apache-forwarder" || [type] == "random-forwarder" { + stdout { + codec => rubydebug + } } } \ No newline at end of file diff --git a/logstash/shipper/ssl/selfsigned.crt b/logstash/shipper/ssl/selfsigned.crt new file mode 100644 index 0000000..216b191 --- /dev/null +++ b/logstash/shipper/ssl/selfsigned.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDEjCCAfqgAwIBAgIQPD0a5WB48xwmuFt+FoT1KDANBgkqhkiG9w0BAQsFADAo +MRQwEgYDVQQKEwtMb2cgQ291cmllcjEQMA4GA1UEAxMHc2hpcHBlcjAeFw0xNzAz +MTcxNDE1NDNaFw0yNzAzMTUxNDE1NDNaMCgxFDASBgNVBAoTC0xvZyBDb3VyaWVy +MRAwDgYDVQQDEwdzaGlwcGVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC +AQEA5lbWxKD/8lYzglubCldiG95jTSOXFbFc11BpH0OkDsoy6uZWyklfBCGkwTJn +8uZRK0RVV02I7ndhzNm9pVD/1wAClOtNnRs3doUuwjdeQCX/tybujUyQWz63irzb +fWBB70XxgknddKWy4fP4Xu54wNLoaWvTTwRxejTWmrvEDz7PpqlWz7mhBXsxFy09 +W5aQpaeBlFR5Pdg+C7yXuTL2oAENQWVMgHpJZqaYB7Vll/JWaHzbbH8kSer9/qIN +Qx9+RWO1SxtuKcpu40NdavT0Km1ZYc3p8i3OOZAJ4pfcB7g7RvUdROElKiO4hWou +Wty046pNWYhum49+zM8r0Q9fSQIDAQABozgwNjAOBgNVHQ8BAf8EBAMCAqQwEwYD +VR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsF +AAOCAQEAO3KmeN/KOFm4OCPKmd5xbZ5I6d/A4V+5lscrFG9tqmF+Ax/hA1RUrnSv +DPk4YSv8zW8tGrZpX3iLGGX7giqT6EcKH+leyyU0pvO2oUvs+R0cHu4kY7XknUHm +Lx3QovDap4+uZIZdawI23fGyKSa7PUybKc1fuxrwZNns3zVc+Lp+iSAfsPWI43kX +ZR0WlMzXIr8EVef+Hz2xLdDsuiwUVrBSWH+tn3pedwPjeEO/YgH+CV899hWlDBl+ +1r8+S11s+Ur+IMuydygLds8QDgLJlFmJVZmvifs+fEd6jPu34CsvA6tF0YqW03iw +Nmnzx5bMkV9Iffg/rdpYapPIV877DQ== +-----END CERTIFICATE----- diff --git a/logstash/shipper/ssl/selfsigned.key b/logstash/shipper/ssl/selfsigned.key new file mode 100644 index 0000000..4d00c20 --- /dev/null +++ b/logstash/shipper/ssl/selfsigned.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA5lbWxKD/8lYzglubCldiG95jTSOXFbFc11BpH0OkDsoy6uZW +yklfBCGkwTJn8uZRK0RVV02I7ndhzNm9pVD/1wAClOtNnRs3doUuwjdeQCX/tybu +jUyQWz63irzbfWBB70XxgknddKWy4fP4Xu54wNLoaWvTTwRxejTWmrvEDz7PpqlW +z7mhBXsxFy09W5aQpaeBlFR5Pdg+C7yXuTL2oAENQWVMgHpJZqaYB7Vll/JWaHzb +bH8kSer9/qINQx9+RWO1SxtuKcpu40NdavT0Km1ZYc3p8i3OOZAJ4pfcB7g7RvUd +ROElKiO4hWouWty046pNWYhum49+zM8r0Q9fSQIDAQABAoIBAF3BsyHONuA5XjYN +e4o2D2UwnFLlzbWywAuUW3WyXrD2dMtQfrtQKjDQUPOixNUGErv90I78rGQEi/aK +GruTwM+O3X4oWSs9zNGdXG8Jgn+x9+hhHHC/XXMGkEcUoHLkH3J5kdhqoI2+dJWs +bMiXYOd78cYXcaeUM2x+3WuWSjO5o6zNDka4RnwFbVk2k2Zay1EsSzjb+9g78Y+C +V+bWYz/o5JdWue5ZmnlvWLS/l8YmlYFaIiHePcbQUtiLDGpIB9LYn63lijRIsDdJ +S+zkxLJfnE9h8b3eViv2h32Ysxd0HCz8on3U6WpvtJ1zTSLZM/wps+clfvHAQxUS +ad1DJBECgYEA9y3cbh62TX9YuDwVNXSpuszAuqSHxXvVo1AQMux8GQnOGyuBwwww +0Nj6nAUOLdIIPhs5fY88D1UJa6TH3s4lnRFURrYa5ueeE0DRU9fVDJ5nnwcPlaXg +bLVRtW7oP3xnFYsB3MQtMANQoHXxqHeO0ttwUI7RhydSY8ctLoZ0cI8CgYEA7o8i +Z5l+9TSQZefqa/eGhOPsEhYnyWfha9R42KOUkPvEu6awgk3UAAdgzY/MN636Gv1O +shWcJGC6K8/pQhkc3jhRICJUZutkUh1r5zKdOKp/lFyGqHF1r7nwhouPg4PDMot1 +z0bv2Q8XRxKoXcDZWj7pQog3LAUKWbub7z677qcCgYEAycm40HVmInI5/X2fWtp4 +zbTmCfjTllb4G7D66HfC/7XPesLT1mWXXIgmX0SJRMLYU8tp7aUHRQUJAcWuHahx +cH57LJbx81tW4yThPu9OUFTpBxx+jo6yXkzP/awjEeD/TskCpjXJJg8uvTBMLloc +UOswD7PoPcvOKQKrUfYWDOUCgYA0rOhU5eDRtfyYc+AgXPSYDVfIGppIb6anz6+r +bltUtDMZmFHrQ9Im+3oVicX3GXRbV0l0Ky9iHPWezPbdgHwOWXMYXFw0qY9qTlLQ +Jy3uID+xe8jEEuGAntcMbm6PBJ4qX/7weOJnFFz4dZc1tGqJ8dBBv6AnGSnS7Ikf +B8QpiwKBgDW+HqJQTwEZMGLHLq3GNlgQqzDQrSRhGMyueeUK1VYPNRpAQQO/KZy/ +7BjjjJswdoF3gZqGBciKYGfqRvojyNOijadJnicuoVNcLqxtZCB4PbFLUfpz00fM +9AQkK0ejTa2/Xh0hF19SqdaK5iiEmFbrDNxW/EastBquOYPAiC1m +-----END RSA PRIVATE KEY-----