summaryrefslogtreecommitdiffstats
path: root/lass/2configs/monitoring
diff options
context:
space:
mode:
authorlassulus <lass@lassul.us>2017-02-05 00:25:39 +0100
committerlassulus <lass@lassul.us>2017-02-05 00:25:39 +0100
commit0ea991ffe9252041751e6e740c5166e164541928 (patch)
tree146bb3394b51d898c0940072de70e6dcd1d71373 /lass/2configs/monitoring
parent2c484daf3cb78deefdcfe2ef6cececaab397dace (diff)
l 2 monitoring server: add ram & deadman alarm
Diffstat (limited to 'lass/2configs/monitoring')
-rw-r--r--lass/2configs/monitoring/server.nix37
1 files changed, 25 insertions, 12 deletions
diff --git a/lass/2configs/monitoring/server.nix b/lass/2configs/monitoring/server.nix
index 2e1c15ca1..505cb7a17 100644
--- a/lass/2configs/monitoring/server.nix
+++ b/lass/2configs/monitoring/server.nix
@@ -1,9 +1,7 @@
{pkgs, config, ...}:
with import <stockholm/lib>;
{
- services.influxdb = {
- enable = true;
- };
+ services.influxdb.enable = true;
services.influxdb.extraConfig = {
meta.hostname = config.krebs.build.host.name;
@@ -29,24 +27,39 @@ with import <stockholm/lib>;
data="$(${pkgs.jq}/bin/jq -r .message)"
export LOGNAME=prism-alarm
${pkgs.irc-announce}/bin/irc-announce \
- irc.freenode.org 6667 prism-alarm \#krebs-bots "$data" >/dev/null
+ ni.r 6667 prism-alarm \#retiolum "$data" >/dev/null
'';
in {
enable = true;
+ check_db = "telegraf_db";
alarms = {
- test2 = ''
- batch
+ cpu = ''
+ var data = batch
|query(${"'''"}
SELECT mean("usage_user") AS mean
FROM "${config.lass.kapacitor.check_db}"."default"."cpu"
${"'''"})
- .every(3m)
- .period(1m)
+ .period(10m)
+ .every(1m)
+ .groupBy('host')
+ data |alert()
+ .crit(lambda: "mean" > 90)
+ .exec('${echoToIrc}')
+ data |deadman(1.0,5m)
+ .stateChangesOnly()
+ .exec('${echoToIrc}')
+ '';
+ ram = ''
+ var data = batch
+ |query(${"'''"}
+ SELECT mean("used_percent") AS mean
+ FROM "${config.lass.kapacitor.check_db}"."default"."mem"
+ ${"'''"})
+ .period(10m)
+ .every(1m)
.groupBy('host')
- |alert()
- .crit(lambda: "mean" > 90)
- // Whenever we get an alert write it to a file.
- .log('/tmp/alerts.log')
+ data |alert()
+ .crit(lambda: "mean" > 90)
.exec('${echoToIrc}')
'';
};