summaryrefslogtreecommitdiffstats
path: root/makefu/2configs/logging
diff options
context:
space:
mode:
authormakefu <github@syntax-fehler.de>2019-09-25 14:01:32 +0200
committermakefu <github@syntax-fehler.de>2019-09-25 14:01:32 +0200
commitc11c3544abd9ec2702c61b7ae114270791a57371 (patch)
tree0c2408a644e1270777ffc3cb29c0b6e6dca4ec67 /makefu/2configs/logging
parent0fbd5ff5631abf062d9cb318035a794a6d47260d (diff)
ma logging: init server
Diffstat (limited to 'makefu/2configs/logging')
-rw-r--r--makefu/2configs/logging/filter/dnsmasq.conf19
-rw-r--r--makefu/2configs/logging/patterns/dnsmasq15
-rw-r--r--makefu/2configs/logging/server.nix123
3 files changed, 154 insertions, 3 deletions
diff --git a/makefu/2configs/logging/filter/dnsmasq.conf b/makefu/2configs/logging/filter/dnsmasq.conf
new file mode 100644
index 000000000..1570b1c60
--- /dev/null
+++ b/makefu/2configs/logging/filter/dnsmasq.conf
@@ -0,0 +1,19 @@
+
+if ( [program] == "dnsmasq") {
+ grok {
+ patterns_dir => ["${./patterns}"]
+ match => {
+ "message" => [
+ "^%{logdate:LOGDATE} dnsmasq\[[\d]+\]\: query\[[\w]+\] %{domain:DOMAIN} from %{clientip:CLIENTIP}"
+ , "^%{logdate:LOGDATE} dnsmasq\[[\d]+\]\: reply %{domain:DOMAIN} is %{ip:IP}"
+ , "^%{logdate:LOGDATE} dnsmasq\[[\d]+\]\: %{blocklist:BLOCKLIST} %{domain:DOMAIN} is %{ip:IP}"
+ ]
+ }
+ }
+ date {
+ match => [ "LOGDATE", "MMM dd HH:mm:ss", "MMM d HH:mm:ss", "ISO8601" ]
+ }
+ geoip {
+ source => "IP"
+ }
+}
diff --git a/makefu/2configs/logging/patterns/dnsmasq b/makefu/2configs/logging/patterns/dnsmasq
new file mode 100644
index 000000000..c1e700d5e
--- /dev/null
+++ b/makefu/2configs/logging/patterns/dnsmasq
@@ -0,0 +1,15 @@
+BLOCKLIST [\/\w\.]+
+DOMAIN [\w\.\-]+
+DNSID \d+
+PORT \d+
+DNSRESPONSE cached|reply|forwarded|query
+# TODO: there are some strange responses for certain queries like <CNAME> or ...
+IPORWORD %{IP}|[<>\.\/\w>]+
+
+# TODO use public suffix list by mozilla
+TLD [a-z]{2,63}
+# matches CCSLD and TLD together (e.g. co.uk )
+CCSLD_TLD [a-z]+\.uk
+# actually after a CCTLD this would be the third level domain ...
+PUBLIC_SUFFIX (xn--)?%{FUNCTIONAL_SLD}\.(%{CCSLD_TLD}|%{TLD})
+FUNCTIONAL_SLD [a-z0-9-]{1,63}
diff --git a/makefu/2configs/logging/server.nix b/makefu/2configs/logging/server.nix
index 90f8e6680..f2fccec25 100644
--- a/makefu/2configs/logging/server.nix
+++ b/makefu/2configs/logging/server.nix
@@ -1,18 +1,29 @@
{pkgs, config, ...}:
-with import <stockholm/lib>;
let
es-port = 9200;
kibana-port = 5601;
+ primaryName = "log.${config.krebs.build.host.name}";
+ serverAliases = [ "${primaryName}.r" "${primaryName}.lan" ];
in {
+
+ services.nginx.virtualHosts.${primaryName} = {
+ inherit serverAliases;
+ locations."/" = {
+ proxyPass = "http://localhost:5601/";
+ extraConfig = ''
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ '';
+ };
+ };
services.elasticsearch = {
enable = true;
- listenAddress = "0.0.0.0";
port = es-port;
};
services.kibana = {
enable = true;
- listenAddress = "0.0.0.0";
port = kibana-port;
};
@@ -20,4 +31,110 @@ in {
iptables -A INPUT -i retiolum -p tcp --dport ${toString es-port} -j ACCEPT
iptables -A INPUT -i retiolum -p tcp --dport ${toString kibana-port} -j ACCEPT
'';
+
+ # send logs directly to elasticsearch
+ services.journalbeat = {
+ enable = true;
+ package = pkgs.journalbeat7;
+ extraConfig = ''
+ logging:
+ to_syslog: true
+ level: info
+ metrics.enabled: false
+ template.enabled: false
+ output.logstash:
+ hosts: [ "127.0.0.1:5044" ]
+ template.enabled: false
+ index: journalbeat
+ journalbeat.inputs:
+ - paths: []
+ seek: cursor
+ '';
+ };
+
+ services.logstash = {
+ enable = true;
+ # package = pkgs.logstash5;
+ # plugins = [ pkgs.logstash-contrib ];
+ inputConfig =
+ ''
+ syslog {
+ timezone => "Etc/UTC"
+ }
+ beats {
+ port => 5044
+ }
+ '';
+ filterConfig =
+ ''
+ # Assume Beats
+ if [syslog] {
+ mutate {
+ add_field => { "program" => "%{[syslog][identifier]}" }
+ }
+ }
+ '' +
+ ''
+ if ![program] {
+ mutate {
+ add_field => { "program" => "unknown" }
+ }
+ }
+ '' +
+ ''
+ if ([program] == "logstash") {
+ drop {}
+ }
+ '' +
+ ''
+ if ( [program] == "dnsmasq") {
+ grok {
+ patterns_dir => ["${./patterns}"]
+ match => {
+ "message" => [
+ "^%{DNSID:dnsid} %{IP:client}/%{PORT} %{DNSRESPONSE:dnstype}\[[\w]+\] %{DOMAIN:domain} from %{IP}"
+ , "^%{DNSID:dnsid} %{IP:client}/%{PORT} %{DNSRESPONSE:dnstype} %{DOMAIN:domain} is %{IPORWORD:resolved_ip}"
+ , "^%{DNSID:dnsid} %{IP:client}/%{PORT} %{DNSRESPONSE:dnstype} %{DOMAIN:domain} to %{IP:upstream_dns}"
+ ]
+ }
+ }
+ if [resolved_ip] {
+ geoip {
+ source => "resolved_ip"
+ }
+ }
+ mutate {
+ rename => { "host" => "syslog_host" }
+ }
+ # Target is to parse the the first and second significant part of the domain
+ grok {
+ patterns_dir => ["${./patterns}"]
+ match => { "domain" => [ "%{PUBLIC_SUFFIX:dns_suffix}$" ] }
+ }
+ if [client] {
+ mutate { copy => { "client" => "clientip" } }
+ dns {
+ reverse => [ "client"]
+ action => "replace"
+ hostsfile => [ "/etc/hosts" ]
+ hit_cache_ttl => 1600
+ failed_cache_ttl => 60
+ }
+ }
+ }
+ '' + ''
+ if ( [program] == "proftpd") {
+ kv {
+ field_split => " "
+ }
+ }
+ '';
+ outputConfig =
+ ''
+ #stdout {
+ # codec => rubydebug
+ #}
+ elasticsearch { }
+ '';
+ };
}