summaryrefslogtreecommitdiffstats
path: root/makefu/2configs/stats
diff options
context:
space:
mode:
authortv <tv@krebsco.de>2017-06-04 06:20:54 +0200
committertv <tv@krebsco.de>2017-06-04 06:20:54 +0200
commit811ceaa243bf5241ca1189871c4426240962f04d (patch)
treef46006567b5f9279ebd9cb23de3eadb508f83c54 /makefu/2configs/stats
parent4f58b884dda57db8106768a22a206d6605d6e3e5 (diff)
parente50bc4f3eb3dac13bba4ae6158e839a52455c3c3 (diff)
Merge remote-tracking branch 'prism/master'
Diffstat (limited to 'makefu/2configs/stats')
-rw-r--r--makefu/2configs/stats/client.nix60
-rw-r--r--makefu/2configs/stats/external/aralast.nix38
-rw-r--r--makefu/2configs/stats/server.nix84
3 files changed, 182 insertions, 0 deletions
diff --git a/makefu/2configs/stats/client.nix b/makefu/2configs/stats/client.nix
new file mode 100644
index 000000000..dd6ddddaf
--- /dev/null
+++ b/makefu/2configs/stats/client.nix
@@ -0,0 +1,60 @@
+{pkgs, config, ...}:
+{
+ services.collectd = {
+ enable = true;
+ autoLoadPlugin = true;
+ extraConfig = ''
+ Hostname ${config.krebs.build.host.name}
+ LoadPlugin load
+ LoadPlugin disk
+ LoadPlugin memory
+ LoadPlugin df
+ Interval 30.0
+
+ LoadPlugin interface
+ <Plugin "interface">
+ Interface "*Link"
+ Interface "lo"
+ Interface "vboxnet*"
+ Interface "virbr*"
+ IgnoreSelected true
+ </Plugin>
+
+ LoadPlugin df
+ <Plugin "df">
+ MountPoint "/nix/store"
+ # MountPoint "/run*"
+ # MountPoint "/sys*"
+ # MountPoint "/dev"
+ # MountPoint "/dev/shm"
+ # MountPoint "/tmp"
+ FSType "tmpfs"
+ FSType "binfmt_misc"
+ FSType "debugfs"
+ FSType "mqueue"
+ FSType "hugetlbfs"
+ FSType "systemd-1"
+ FSType "cgroup"
+ FSType "securityfs"
+ FSType "ramfs"
+ FSType "proc"
+ FSType "devpts"
+ FSType "devtmpfs"
+ MountPoint "/var/lib/docker/devicemapper"
+ IgnoreSelected true
+ </Plugin>
+
+ LoadPlugin cpu
+ <Plugin cpu>
+ ReportByCpu true
+ ReportByState true
+ ValuesPercentage true
+ </Plugin>
+
+ LoadPlugin network
+ <Plugin "network">
+ Server "${config.makefu.stats-server}" "25826"
+ </Plugin>
+ '';
+ };
+}
diff --git a/makefu/2configs/stats/external/aralast.nix b/makefu/2configs/stats/external/aralast.nix
new file mode 100644
index 000000000..870db99a8
--- /dev/null
+++ b/makefu/2configs/stats/external/aralast.nix
@@ -0,0 +1,38 @@
+{ config, lib, pkgs, ... }:
+
+with import <stockholm/lib>;
+let
+ pkg = pkgs.stdenv.mkDerivation {
+ name = "aralast-master";
+ src = pkgs.fetchFromGitHub {
+ owner = "makefu";
+ repo = "aralast";
+ rev = "7121598";
+ sha256 = "0vw027c698h9b69ksid5p3pji9960hd7n9xi4arrax0vfkwryb4m";
+ };
+ installPhase = ''
+ install -m755 -D aralast.sh $out/bin/aralast
+ '';
+ };
+in {
+ systemd.services.aralast = {
+ description = "periodically fetch aramark";
+ path = [
+ pkgs.curl
+ pkgs.gnugrep
+ pkgs.gnused
+ ];
+ wantedBy = [ "multi-user.target" ];
+ environment = {
+ INFLUX_HOST = "localhost";
+ INFLUX_PORT = "8086";
+ };
+ # every 10 seconds when the cantina is open
+ startAt = "Mon,Tue,Wed,Thu,Fri *-*-* 6,7,8,9,10,11,12,13,14,15:*:0,15,30,45";
+ serviceConfig = {
+ User = "nobody";
+ ExecStart = "${pkg}/bin/aralast";
+ PrivateTmp = true;
+ };
+ };
+}
diff --git a/makefu/2configs/stats/server.nix b/makefu/2configs/stats/server.nix
new file mode 100644
index 000000000..602fcc6d0
--- /dev/null
+++ b/makefu/2configs/stats/server.nix
@@ -0,0 +1,84 @@
+{pkgs, config, ...}:
+
+with import <stockholm/lib>;
+let
+ collectd-port = 25826;
+ influx-port = 8086;
+ grafana-port = 3000; # TODO nginx forward
+ db = "collectd_db";
+ logging-interface = config.makefu.server.primary-itf;
+in {
+ services.grafana.enable = true;
+ services.grafana.addr = "0.0.0.0";
+
+ services.influxdb.enable = true;
+ # redirect grafana to stats.makefu.r
+ services.nginx.enable = true;
+ services.nginx.virtualHosts."stats.makefu.r".locations."/".proxyPass = "http://localhost:3000";
+ # forward these via nginx
+ services.influxdb.extraConfig = {
+ meta.hostname = config.krebs.build.host.name;
+ # meta.logging-enabled = true;
+ http.bind-address = ":${toString influx-port}";
+ admin.bind-address = ":8083";
+ monitoring = {
+ enabled = false;
+ # write-interval = "24h";
+ };
+ collectd = [{
+ enabled = true;
+ typesdb = "${pkgs.collectd}/share/collectd/types.db";
+ database = db;
+ port = collectd-port;
+ }];
+ };
+ krebs.kapacitor =
+ let
+ echoToIrc = pkgs.writeDash "echo_irc" ''
+ set -euf
+ data="$(${pkgs.jq}/bin/jq -r .message)"
+ export LOGNAME=malarm
+ ${pkgs.irc-announce}/bin/irc-announce \
+ irc.freenode.org 6667 malarm \#krebs-bots "$data" >/dev/null
+ '';
+ in {
+ enable = true;
+ alarms = {
+ cpu_deadman.database = db;
+ cpu_deadman.text = ''
+ var data = batch
+ |query(${"'''"}
+ SELECT mean("value") AS mean
+ FROM "collectd_db"."default"."cpu_value"
+ WHERE "type_instance" = 'idle' AND "type" = 'percent' fill(0)
+ ${"'''"})
+ .period(10m)
+ .every(1m)
+ .groupBy('host')
+ data |alert()
+ .crit(lambda: "mean" < 50)
+ .stateChangesOnly()
+ .exec('${echoToIrc}')
+ data |deadman(1.0,5m)
+ .stateChangesOnly()
+ .exec('${echoToIrc}')
+ '';
+ };
+
+ };
+ networking.firewall.extraCommands = ''
+ iptables -A INPUT -i retiolum -p udp --dport ${toString collectd-port} -j ACCEPT
+ iptables -A INPUT -i retiolum -p tcp --dport ${toString influx-port} -j ACCEPT
+ iptables -A INPUT -i retiolum -p tcp --dport ${toString grafana-port} -j ACCEPT
+ iptables -A INPUT -i ${logging-interface} -p udp --dport ${toString collectd-port} -j ACCEPT
+ iptables -A INPUT -i ${logging-interface} -p tcp --dport ${toString influx-port} -j ACCEPT
+ iptables -A INPUT -i ${logging-interface} -p tcp --dport ${toString grafana-port} -j ACCEPT
+
+ ip6tables -A INPUT -i retiolum -p udp --dport ${toString collectd-port} -j ACCEPT
+ ip6tables -A INPUT -i retiolum -p tcp --dport ${toString influx-port} -j ACCEPT
+ ip6tables -A INPUT -i retiolum -p tcp --dport ${toString grafana-port} -j ACCEPT
+ ip6tables -A INPUT -i ${logging-interface} -p udp --dport ${toString collectd-port} -j ACCEPT
+ ip6tables -A INPUT -i ${logging-interface} -p tcp --dport ${toString influx-port} -j ACCEPT
+ ip6tables -A INPUT -i ${logging-interface} -p tcp --dport ${toString grafana-port} -j ACCEPT
+ '';
+}