puyak: enable blackbox prometheus exporter, use it to ping internal and external hosts

4 files changed, 66 insertions, 23 deletions

diff --git a/krebs/1systems/puyak/config.nix b/krebs/1systems/puyak/config.nix
index eb1256d69..a50d2eab4 100644
--- a/krebs/1systems/puyak/config.nix
+++ b/krebs/1systems/puyak/config.nix
@@ -16,6 +16,7 @@
     <stockholm/krebs/2configs/shack/ssh-keys.nix>
     <stockholm/krebs/2configs/shack/prometheus/node.nix>
     <stockholm/krebs/2configs/shack/prometheus/server.nix>
+    <stockholm/krebs/2configs/shack/prometheus/blackbox.nix>
     <stockholm/krebs/2configs/shack/prometheus/unifi.nix>
     <stockholm/krebs/2configs/shack/gitlab-runner.nix>
 
diff --git a/krebs/2configs/shack/glados/default.nix b/krebs/2configs/shack/glados/default.nix
index b6bcc8673..43d1ae3f3 100644
--- a/krebs/2configs/shack/glados/default.nix
+++ b/krebs/2configs/shack/glados/default.nix
@@ -29,8 +29,7 @@ in {
       };
     in {
     enable = true;
-    package = (pkgs.home-assistant.overrideAttrs (old: {
-      # TODO: find correct python package
+    package = (pkgs.home-assistant.overrideAttrs (old: { # TODO: find correct python package
       installCheckPhase = ''
         echo LOLLLLLLLLLLLLLL
       '';
@@ -70,6 +69,8 @@ in {
       #    source = "hass";
       #  };
       #};
+      esphome = {};
+      api = {};
       mqtt = {
         broker = "localhost";
         port = 1883;
@@ -130,11 +131,9 @@ in {
         trusted_proxies = "127.0.0.1";
       };
       #conversation = {};
-      history = {};
+      # history = {};
       #logbook = {};
-      logger = {
-        default = "info";
-      };
+      logger.default = "info";
       #recorder = {};
       tts = [
         { platform = "google_translate";
@@ -143,10 +142,6 @@ in {
           cache = true;
           time_memory = 57600;
         }
-        { platform = "picotts";
-          language = "de-DE";
-          service_name = "pico";
-        }
       ];
       sun = {};
 
diff --git a/krebs/2configs/shack/prometheus/blackbox.nix b/krebs/2configs/shack/prometheus/blackbox.nix
new file mode 100644
index 000000000..82ce003e8
--- /dev/null
+++ b/krebs/2configs/shack/prometheus/blackbox.nix
@@ -0,0 +1,19 @@
+{pkgs, ... }:
+{
+  systemd.services.prometheus-blackbox-exporter.serviceConfig = {
+    CapabilityBoundingSet = ["CAP_NET_RAW"]; # icmp allow
+    AmbientCapabilities = ["CAP_NET_RAW"];
+  };
+  services.prometheus.exporters.blackbox = {
+    enable = true;
+    # openFirewall = true; # not requred if running on the same host as prometheus
+    port = 9115;
+    configFile = pkgs.writeText "icmp" ''
+      modules:
+        icmp:
+          prober: icmp
+          icmp:
+            preferred_ip_protocol: ip4
+    '';
+  };
+}
diff --git a/krebs/2configs/shack/prometheus/server.nix b/krebs/2configs/shack/prometheus/server.nix
index f5d2e7640..c088a3b08 100644
--- a/krebs/2configs/shack/prometheus/server.nix
+++ b/krebs/2configs/shack/prometheus/server.nix
@@ -18,16 +18,6 @@
     };
     prometheus = {
       enable = true;
-      extraFlags = [
-        "-storage.local.retention 720h"
-        "-storage.local.series-file-shrink-ratio 0.3"
-        "-storage.local.memory-chunks 2097152"
-        "-storage.local.max-chunks-to-persist 1048576"
-        "-storage.local.index-cache-size.fingerprint-to-metric 2097152"
-        "-storage.local.index-cache-size.fingerprint-to-timerange 1048576"
-        "-storage.local.index-cache-size.label-name-to-label-values 2097152"
-        "-storage.local.index-cache-size.label-pair-to-fingerprints 41943040"
-      ];
       ruleFiles = lib.singleton (pkgs.writeText "prometheus-rules.yml" (builtins.toJSON {
             groups = lib.singleton {
               name = "mf-alerting-rules";
@@ -41,7 +31,7 @@
           static_configs = [
             {
               targets = [
-                "localhost:9100"
+                "wolf.shack:9100"
               ];
               labels = {
                 alias = "wolf.shack";
@@ -49,7 +39,15 @@
             }
             {
               targets = [
-                "localhost:9130"
+                "infra01.shack:9100"
+              ];
+              labels = {
+                alias = "infra01.shack";
+              };
+            }
+            {
+              targets = [
+                "unifi.shack:9130"
               ];
               labels = {
                 alias = "unifi.shack";
@@ -57,7 +55,7 @@
             }
             {
               targets = [
-                "10.42.22.184:9100" # puyak.shack
+                "puyak.shack:9100" # puyak.shack
               ];
               labels = {
                 alias = "puyak.shack";
@@ -81,6 +79,36 @@
             }
           ];
         }
+        {
+          job_name = "blackbox";
+          metrics_path = "/probe";
+          params.module = [ "icmp" ];
+          static_configs = [
+            {
+              targets = [
+                "google.com"
+                "wolf.shack"
+                "web.de"
+                "10.0.0.1"
+                "licht.shack"
+              ];
+            }
+          ];
+          relabel_configs = [
+            {
+              source_labels = ["__address__"];
+              target_label = "__param_target";
+            }
+            {
+              source_labels = ["__param_target"];
+              target_label = "instance";
+            }
+            {
+              target_label = "__address__";
+              replacement = "127.0.0.1:9115";
+            }
+          ];
+        }
       ];
       alertmanagers = [
         { scheme = "http";