From c22c74b05d0da8febe1fbeec0808f7b66f3920fc Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 9 Jan 2017 15:56:44 +0100 Subject: k 3 m: gum add subdomains --- krebs/3modules/makefu/default.nix | 4 ++++ makefu/6tests/data/secrets/dl.euer.krebsco.de-auth.nix | 1 + 2 files changed, 5 insertions(+) create mode 100644 makefu/6tests/data/secrets/dl.euer.krebsco.de-auth.nix diff --git a/krebs/3modules/makefu/default.nix b/krebs/3modules/makefu/default.nix index c85bf1ccd..9942ac043 100644 --- a/krebs/3modules/makefu/default.nix +++ b/krebs/3modules/makefu/default.nix @@ -453,6 +453,8 @@ TNs2RYfwDy/r6H/hDeB/BSngPouedEVcPwIDAQAB git.euer IN A ${nets.internet.ip4.addr} gum IN A ${nets.internet.ip4.addr} cgit.euer IN A ${nets.internet.ip4.addr} + o.euer IN A ${nets.internet.ip4.addr} + dl.euer IN A ${nets.internet.ip4.addr} ''; }; nets = rec { @@ -470,6 +472,8 @@ TNs2RYfwDy/r6H/hDeB/BSngPouedEVcPwIDAQAB "gum.r" "gum.retiolum" "cgit.gum.retiolum" + "o.gum.r" + "o.gum.retiolum" "tracker.makefu.r" "tracker.makefu.retiolum" ]; diff --git a/makefu/6tests/data/secrets/dl.euer.krebsco.de-auth.nix b/makefu/6tests/data/secrets/dl.euer.krebsco.de-auth.nix new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/makefu/6tests/data/secrets/dl.euer.krebsco.de-auth.nix @@ -0,0 +1 @@ +{} -- cgit v1.2.3 From b1b24e475939fced39d0d3db4801bd4c9174b042 Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 9 Jan 2017 15:57:32 +0100 Subject: k 3 rtorrent: always restart on-failure does not seem to be enough to save it from crashing --- krebs/3modules/rtorrent.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/krebs/3modules/rtorrent.nix b/krebs/3modules/rtorrent.nix index 0c478aded..d85518993 100644 --- a/krebs/3modules/rtorrent.nix +++ b/krebs/3modules/rtorrent.nix @@ -224,13 +224,14 @@ let cp -f ${configFile} ${cfg.workDir}/.rtorrent.rc ''; ExecStart = "${pkgs.tmux}/bin/tmux new-session -s rt -n rtorrent -d 'PATH=/bin:/usr/bin:${makeBinPath rutorrent-deps} ${cfg.package}/bin/rtorrent'"; + Restart = "always"; + RestartSec = "10"; ## you can simply sudo -u rtorrent tmux a if privateTmp is set to false ## otherwise the tmux session is stored in some private folder in /tmp PrivateTmp = false; WorkingDirectory = cfg.workDir; - Restart = "on-failure"; User = "${cfg.user}"; }; }; -- cgit v1.2.3 From 1ef8cc193faba5dff71e7028c3ed59a81db08b6a Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 9 Jan 2017 15:58:43 +0100 Subject: m 2 backup: add gum to backup --- makefu/2configs/backup.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/makefu/2configs/backup.nix b/makefu/2configs/backup.nix index 38fd93e2b..280dc1df4 100644 --- a/makefu/2configs/backup.nix +++ b/makefu/2configs/backup.nix @@ -29,6 +29,7 @@ let }; in { krebs.backup.plans = { - wry-to-omo_var-www = defaultPull config.krebs.hosts.wry "/"; + wry-to-omo_root = defaultPull config.krebs.hosts.wry "/"; + gum-to-omo_root = defaultPull config.krebs.hosts.gum "/"; }; } -- cgit v1.2.3 From c7f1f7b778ac2a3f9901c71c79f52907be24fc4f Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 9 Jan 2017 15:59:10 +0100 Subject: m 2 elchos/log: do not write to stdout --- makefu/2configs/elchos/log.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/makefu/2configs/elchos/log.nix b/makefu/2configs/elchos/log.nix index 3facd1ceb..34bd2494d 100644 --- a/makefu/2configs/elchos/log.nix +++ b/makefu/2configs/elchos/log.nix @@ -21,9 +21,9 @@ in { } ''; outputConfig = '' - stdout { - codec => rubydebug - } + #stdout { + # codec => rubydebug + #} elasticsearch { } ''; }; -- cgit v1.2.3 From 9e962a9f0eb70c569d6ac3ec1907b42a456ca800 Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 9 Jan 2017 15:59:40 +0100 Subject: m 2 elchos/stats: remove graphite-beacon --- makefu/2configs/elchos/stats.nix | 67 +++++++--------------------------------- 1 file changed, 12 insertions(+), 55 deletions(-) diff --git a/makefu/2configs/elchos/stats.nix b/makefu/2configs/elchos/stats.nix index feb8c5bae..2036b391f 100644 --- a/makefu/2configs/elchos/stats.nix +++ b/makefu/2configs/elchos/stats.nix @@ -7,6 +7,11 @@ with import ; { + networking.firewall = { + allowedTCPPorts = [ 2003 80 443 18080 ]; + allowedUDPPorts = [ 2003 ]; + }; + services.nginx = { enable = mkDefault true; virtualHosts = { @@ -39,58 +44,11 @@ with import ; }; services.graphite = { - beacon = { - enable = true; - config = { - graphite_url = "http://localhost:18080"; - - no_data = "critical"; - loading_error = "normal"; - - prefix = "[elchos]"; - - cli = { - command = ''${pkgs.irc-announce}/bin/irc-announce irc.freenode.org 6667 alert0r \#elchos ' [elchos] ''${level} ''${name} ''${value}' ''; - }; - #smtp = { - # from = "beacon@mors.r"; - # to = [ - # "lass@mors.r" - # ]; - #}; - normal_handlers = [ - # "smtp" - "cli" - ]; - warning_handlers = [ - # "smtp" - "cli" - ]; - critical_handlers = [ - # "smtp" - "cli" - ]; - alerts = let - high-load = hostid: let - host = "elch-${toString hostid}"; in { - name = "high-cpu-load-${host}"; - query = "aliasByNode(perSecond(elchos.${host}.cpu.0.cpu.idle),1)"; - method = "average"; - interval = "1minute"; - logging = "info"; - repeat_interval = "5minute"; - rules = [ - # "warning: < 30.0" - "critical: < 1.0" - ]; - }; - in map high-load [ 1 2 3 4 5 6 7 8 ]; - }; - }; api = { enable = true; - package = pkgs.graphiteApi; - listenAddress = "127.0.0.1"; + # package = pkgs.graphiteApi; + #listenAddress = "127.0.0.1"; + listenAddress = "0.0.0.0"; port = 18080; }; carbon = { @@ -99,8 +57,11 @@ with import ; config = '' [cache] MAX_CACHE_SIZE = inf - MAX_UPDATES_PER_SECOND = 10 + MAX_UPDATES_PER_SECOND = 3 MAX_CREATES_PER_MINUTE = 5000 + LOG_UPDATES = False + LOG_CACHE_HITS = False + LOG_CACHE_QUEUE_SORTS = False ''; storageSchemas = '' [carbon] @@ -122,8 +83,4 @@ with import ; }; }; - networking.firewall = { - allowedTCPPorts = [ 2003 80 443 ]; - allowedUDPPorts = [ 2003 ]; - }; } -- cgit v1.2.3 From 6d8e758ecf7cbcec4e00c7e72fcdc433341612a2 Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 9 Jan 2017 16:00:57 +0100 Subject: m 2 omo-share: expose crypt0 read-only --- makefu/2configs/omo-share.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/makefu/2configs/omo-share.nix b/makefu/2configs/omo-share.nix index d1a9fabd7..8a3eab98a 100644 --- a/makefu/2configs/omo-share.nix +++ b/makefu/2configs/omo-share.nix @@ -42,6 +42,12 @@ in { browseable = "yes"; "guest ok" = "yes"; }; + crypt0 = { + path = "/media/crypt0"; + "read only" = "yes"; + browseable = "yes"; + "guest ok" = "yes"; + }; crypt0-rw = { path = "/media/crypt0/"; "read only" = "no"; -- cgit v1.2.3 From 2f19eccf24e6b936542bbab04a435f3127c65962 Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 9 Jan 2017 16:01:46 +0100 Subject: m: expose torrent-secrets --- makefu/2configs/torrent.nix | 4 ++-- makefu/3modules/default.nix | 2 ++ makefu/3modules/torrent.nix | 13 +++++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 makefu/3modules/torrent.nix diff --git a/makefu/2configs/torrent.nix b/makefu/2configs/torrent.nix index f3bc9091b..69d40094f 100644 --- a/makefu/2configs/torrent.nix +++ b/makefu/2configs/torrent.nix @@ -8,13 +8,13 @@ let peer-port = 51412; web-port = 8112; daemon-port = 58846; - dl-dir = "/var/download"; + dl-dir = config.makefu.dl-dir; in { # prepare secrets krebs.build.source.torrent-secrets.file = if getEnv "dummy_secrets" == "true" then toString - else "/home/makefu/secrets/torrent"; + else config.makefu.torrent-secrets ; users.users = { download = { diff --git a/makefu/3modules/default.nix b/makefu/3modules/default.nix index 16215b27a..546fed109 100644 --- a/makefu/3modules/default.nix +++ b/makefu/3modules/default.nix @@ -8,9 +8,11 @@ _: ./forward-journal.nix ./opentracker.nix ./ps3netsrv.nix + ./logging-config.nix ./server-config.nix ./snapraid.nix ./taskserver.nix + ./torrent.nix ./udpt.nix ./umts.nix ]; diff --git a/makefu/3modules/torrent.nix b/makefu/3modules/torrent.nix new file mode 100644 index 000000000..9f1cf3398 --- /dev/null +++ b/makefu/3modules/torrent.nix @@ -0,0 +1,13 @@ +{config, lib, pkgs, ... }: + +{ + options.makefu.dl-dir = lib.mkOption { + type = lib.types.str; + description = "Default download directory"; + default = "/media/cryptX/torrent"; + }; + options.makefu.torrent-secrets = lib.mkOption { + type = lib.types.str; + default = "/home/makefu/secrets/torrent"; + }; +} -- cgit v1.2.3 From 1ca165d96d21e13f5c62e9a0a45d21ecef984992 Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 9 Jan 2017 16:02:02 +0100 Subject: m 2 logging: init --- makefu/2configs/logging/central-stats-client.nix | 60 ++++++++++++++++++++++++ makefu/2configs/logging/central-stats-server.nix | 36 ++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 makefu/2configs/logging/central-stats-client.nix create mode 100644 makefu/2configs/logging/central-stats-server.nix diff --git a/makefu/2configs/logging/central-stats-client.nix b/makefu/2configs/logging/central-stats-client.nix new file mode 100644 index 000000000..c1365c581 --- /dev/null +++ b/makefu/2configs/logging/central-stats-client.nix @@ -0,0 +1,60 @@ +{pkgs, config, ...}: +{ + services.collectd = { + enable = true; + autoLoadPlugin = true; + extraConfig = '' + Hostname ${config.krebs.build.host.name} + LoadPlugin load + LoadPlugin disk + LoadPlugin memory + LoadPlugin df + Interval 30.0 + + LoadPlugin interface + + Interface "*Link" + Interface "lo" + Interface "vboxnet*" + Interface "virbr*" + IgnoreSelected true + + + LoadPlugin df + + MountPoint "/nix/store" + # MountPoint "/run*" + # MountPoint "/sys*" + # MountPoint "/dev" + # MountPoint "/dev/shm" + # MountPoint "/tmp" + FSType "tmpfs" + FSType "binfmt_misc" + FSType "debugfs" + FSType "mqueue" + FSType "hugetlbfs" + FSType "systemd-1" + FSType "cgroup" + FSType "securityfs" + FSType "ramfs" + FSType "proc" + FSType "devpts" + FSType "devtmpfs" + MountPoint "/var/lib/docker/devicemapper" + IgnoreSelected true + + + LoadPlugin cpu + + ReportByCpu true + ReportByState true + ValuesPercentage true + + + LoadPlugin network + + Server "${config.makefu.stats-server}" "25826" + + ''; + }; +} diff --git a/makefu/2configs/logging/central-stats-server.nix b/makefu/2configs/logging/central-stats-server.nix new file mode 100644 index 000000000..eb4e2a344 --- /dev/null +++ b/makefu/2configs/logging/central-stats-server.nix @@ -0,0 +1,36 @@ +{pkgs, config, ...}: + +with import ; +let + collectd-port = 25826; + influx-port = 8086; + grafana-port = 3000; # TODO nginx forward +in { + services.grafana.enable = true; + services.grafana.addr = "0.0.0.0"; + + services.influxdb.enable = true; + + # forward these via nginx + services.influxdb.extraConfig = { + meta.hostname = config.krebs.build.host.name; + # meta.logging-enabled = true; + http.bind-address = ":${toString influx-port}"; + admin.bind-address = ":8083"; + monitoring = { + enabled = false; + # write-interval = "24h"; + }; + collectd = [{ + enabled = true; + typesdb = "${pkgs.collectd}/share/collectd/types.db"; + database = "collectd_db"; + port = collectd-port; + }]; + }; + networking.firewall.extraCommands = '' + iptables -A INPUT -i retiolum -p udp --dport ${toString collectd-port} -j ACCEPT + iptables -A INPUT -i retiolum -p tcp --dport ${toString influx-port} -j ACCEPT + iptables -A INPUT -i retiolum -p tcp --dport ${toString grafana-port} -j ACCEPT + ''; +} -- cgit v1.2.3 From eb208085f011aa48bae73b98caa1f4fccae74ab6 Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 9 Jan 2017 16:02:20 +0100 Subject: m 2 nginx/share-download: expose dl-dir via nginx --- makefu/2configs/nginx/share-download.nix | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 makefu/2configs/nginx/share-download.nix diff --git a/makefu/2configs/nginx/share-download.nix b/makefu/2configs/nginx/share-download.nix new file mode 100644 index 000000000..65c44b294 --- /dev/null +++ b/makefu/2configs/nginx/share-download.nix @@ -0,0 +1,18 @@ +{ config, lib, pkgs, ... }: + +with import ; +{ + services.nginx = { + enable = mkDefault true; + recommendedGzipSettings = true; + recommendedOptimisation = true; + virtualHosts."dl.euer.krebsco.de" = { + root = config.makefu.dl-dir; + extraConfig = "autoindex on;"; + forceSSL = true; + enableSSL = true; + enableACME = true; + basicAuth = import ; + }; + }; +} -- cgit v1.2.3 From 66c1ef89b7e950105ddbd918f1900f5625c08c7b Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 9 Jan 2017 16:03:26 +0100 Subject: m 3 logging-config: init --- makefu/3modules/logging-config.nix | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 makefu/3modules/logging-config.nix diff --git a/makefu/3modules/logging-config.nix b/makefu/3modules/logging-config.nix new file mode 100644 index 000000000..cd6924f61 --- /dev/null +++ b/makefu/3modules/logging-config.nix @@ -0,0 +1,11 @@ +{config, lib, pkgs, ... }: + +with import ; +{ + options.makefu.stats-server = lib.mkOption { + type = types.str; + default = "omo.retiolum"; + description = "Central stats server (collectd)"; + }; +} + -- cgit v1.2.3 From 0695846f3c3d474eb8ee7d4b902e14676fd1a8d8 Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 9 Jan 2017 16:03:36 +0100 Subject: m 2 logging: retab --- makefu/2configs/logging/central-stats-client.nix | 46 ++++++++++++------------ makefu/2configs/logging/central-stats-server.nix | 8 ++--- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/makefu/2configs/logging/central-stats-client.nix b/makefu/2configs/logging/central-stats-client.nix index c1365c581..dd6ddddaf 100644 --- a/makefu/2configs/logging/central-stats-client.nix +++ b/makefu/2configs/logging/central-stats-client.nix @@ -1,26 +1,26 @@ {pkgs, config, ...}: { - services.collectd = { + services.collectd = { enable = true; autoLoadPlugin = true; extraConfig = '' - Hostname ${config.krebs.build.host.name} - LoadPlugin load - LoadPlugin disk - LoadPlugin memory - LoadPlugin df - Interval 30.0 + Hostname ${config.krebs.build.host.name} + LoadPlugin load + LoadPlugin disk + LoadPlugin memory + LoadPlugin df + Interval 30.0 - LoadPlugin interface - - Interface "*Link" - Interface "lo" - Interface "vboxnet*" - Interface "virbr*" - IgnoreSelected true - + LoadPlugin interface + + Interface "*Link" + Interface "lo" + Interface "vboxnet*" + Interface "virbr*" + IgnoreSelected true + - LoadPlugin df + LoadPlugin df MountPoint "/nix/store" # MountPoint "/run*" @@ -41,15 +41,15 @@ FSType "devpts" FSType "devtmpfs" MountPoint "/var/lib/docker/devicemapper" - IgnoreSelected true + IgnoreSelected true - LoadPlugin cpu - - ReportByCpu true - ReportByState true - ValuesPercentage true - + LoadPlugin cpu + + ReportByCpu true + ReportByState true + ValuesPercentage true + LoadPlugin network diff --git a/makefu/2configs/logging/central-stats-server.nix b/makefu/2configs/logging/central-stats-server.nix index eb4e2a344..69e37e493 100644 --- a/makefu/2configs/logging/central-stats-server.nix +++ b/makefu/2configs/logging/central-stats-server.nix @@ -2,11 +2,11 @@ with import ; let - collectd-port = 25826; - influx-port = 8086; + collectd-port = 25826; + influx-port = 8086; grafana-port = 3000; # TODO nginx forward in { - services.grafana.enable = true; + services.grafana.enable = true; services.grafana.addr = "0.0.0.0"; services.influxdb.enable = true; @@ -21,7 +21,7 @@ in { enabled = false; # write-interval = "24h"; }; - collectd = [{ + collectd = [{ enabled = true; typesdb = "${pkgs.collectd}/share/collectd/types.db"; database = "collectd_db"; -- cgit v1.2.3 From a510069d86bd00d9ff348bdad321294cdf42dafb Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 10 Jan 2017 11:34:44 +0100 Subject: m 1 wry: add central-stats-client --- makefu/1systems/wry.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/makefu/1systems/wry.nix b/makefu/1systems/wry.nix index 6290ff6e9..03114c0e6 100644 --- a/makefu/1systems/wry.nix +++ b/makefu/1systems/wry.nix @@ -24,10 +24,10 @@ in { # ../2configs/nginx/euer.test.nix # collectd - # ../2configs/collectd/collectd-base.nix + ../2configs/logging/central-stats-client.nix ../2configs/tinc/retiolum.nix - ../2configs/torrent.nix + # ../2configs/torrent.nix ]; krebs.build.host = config.krebs.hosts.wry; -- cgit v1.2.3