summaryrefslogtreecommitdiffstats
path: root/lass/2configs/monitoring/server.nix
blob: 335820bc71710b2b69ee188636feb0c3a004f42b (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
{pkgs, config, ...}:
with import <stockholm/lib>;
{
  services.influxdb = {
    enable = true;
  };

  services.influxdb.extraConfig = {
    meta.hostname = config.krebs.build.host.name;
    # meta.logging-enabled = true;
    http.bind-address = ":8086";
    admin.bind-address = ":8083";
    monitoring = {
      enabled = false;
      # write-interval = "24h";
    };
  };

  lass.kapacitor =
    let
      echoToIrc = pkgs.writeDash "echo_irc" ''
        set -euf
        data="$(${pkgs.jq}/bin/jq -r .message)"
        export LOGNAME=prism-alarm
        ${pkgs.irc-announce}/bin/irc-announce \
          irc.freenode.org 6667 prism-alarm \#krebs-bots "$data" >/dev/null
      '';
    in {
      enable = true;
      alarms = {
        test2 = ''
          batch
            |query(${"'''"}
              SELECT mean("usage_user") AS mean
              FROM "${config.lass.kapacitor.check_db}"."default"."cpu"
            ${"'''"})
            .every(3m)
            .period(1m)
            .groupBy('host')
            |alert()
              .crit(lambda: "mean" >  90)
              // Whenever we get an alert write it to a file.
              .log('/tmp/alerts.log')
              .exec('${echoToIrc}')
        '';
      };
  };

  krebs.iptables.tables.filter.INPUT.rules = [
    { predicate = "-p tcp -i retiolum --dport 8086"; target = "ACCEPT"; }
    { predicate = "-p tcp -i retiolum --dport 3000"; target = "ACCEPT"; }
  ];
  services.grafana = {
    enable = true;
    addr = "0.0.0.0";
    auth.anonymous.enable = true;
    security = import <secrets/grafana_security.nix>; # { AdminUser = ""; adminPassword = ""}
  };
}