summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--.rsync-filter2
-rw-r--r--Makefile76
-rw-r--r--default.nix32
-rw-r--r--krebs/3modules/backup.nix22
-rw-r--r--krebs/3modules/build.nix171
-rw-r--r--krebs/3modules/git.nix201
-rw-r--r--krebs/3modules/tv/default.nix1
-rw-r--r--krebs/3modules/urlwatch.nix46
-rw-r--r--krebs/4lib/default.nix1
-rw-r--r--krebs/4lib/types.nix4
-rw-r--r--krebs/5pkgs/Reaktor/plugins.nix2
-rw-r--r--krebs/5pkgs/cac-api/default.nix42
-rw-r--r--krebs/5pkgs/cac-cert/cac.pem (renamed from krebs/5pkgs/test/infest-cac-centos7/panel.cloudatcost.com.crt)0
-rw-r--r--krebs/5pkgs/cac-cert/default.nix2
-rw-r--r--krebs/5pkgs/cac/default.nix39
-rw-r--r--krebs/5pkgs/get/default.nix6
-rw-r--r--krebs/5pkgs/lentil/default.nix15
-rw-r--r--krebs/5pkgs/lentil/syntaxes.patch11
-rw-r--r--krebs/5pkgs/much/default.nix2
-rw-r--r--krebs/5pkgs/test/infest-cac-centos7/default.nix22
-rwxr-xr-xkrebs/5pkgs/test/infest-cac-centos7/notes12
-rw-r--r--krebs/5pkgs/urlwatch/default.nix39
-rw-r--r--krebs/5pkgs/with-tmpdir/default.nix29
-rw-r--r--krebs/default.nix105
-rw-r--r--makefu/1systems/omo.nix2
-rw-r--r--makefu/1systems/vbob.nix4
-rw-r--r--makefu/2configs/backup.nix30
-rw-r--r--makefu/2configs/default.nix22
-rw-r--r--makefu/2configs/unstable-sources.nix2
l---------nixpkgs/default.nix1
-rw-r--r--nixpkgs/krebs0
l---------nixpkgs/lib1
-rw-r--r--nixpkgs/nixos/default.nix1
l---------nixpkgs/nixos/lib1
l---------nixpkgs/nixos/modules1
l---------nixpkgs/pkgs1
l---------root1
-rw-r--r--shared/2configs/buildbot-standalone.nix2
-rw-r--r--tv/1systems/cd.nix43
-rw-r--r--tv/1systems/mkdir.nix1
-rw-r--r--tv/1systems/nomic.nix41
-rw-r--r--tv/1systems/rmdir.nix1
-rw-r--r--tv/1systems/wu.nix43
-rw-r--r--tv/1systems/xu.nix44
-rw-r--r--tv/2configs/backup.nix40
-rw-r--r--tv/2configs/consul-client.nix9
-rw-r--r--tv/2configs/consul-server.nix21
-rw-r--r--tv/2configs/default.nix52
-rw-r--r--tv/2configs/exim-retiolum.nix8
-rw-r--r--tv/2configs/exim-smarthost.nix5
-rw-r--r--tv/2configs/git.nix9
-rw-r--r--tv/2configs/hw/AO753.nix9
-rw-r--r--tv/2configs/hw/x220.nix1
-rw-r--r--tv/2configs/nginx-public_html.nix15
-rw-r--r--tv/2configs/pulse.nix32
-rw-r--r--tv/2configs/retiolum.nix17
-rw-r--r--tv/2configs/vim.nix9
-rw-r--r--tv/2configs/xserver/default.nix1
-rw-r--r--tv/3modules/consul.nix118
-rw-r--r--tv/3modules/default.nix1
-rw-r--r--tv/5pkgs/default.nix3
-rw-r--r--tv/5pkgs/ejabberd/default.nix28
62 files changed, 772 insertions, 730 deletions
diff --git a/.rsync-filter b/.rsync-filter
new file mode 100644
index 000000000..d7657cd00
--- /dev/null
+++ b/.rsync-filter
@@ -0,0 +1,2 @@
+- /.git
+- /.graveyard
diff --git a/Makefile b/Makefile
index 5b898c54c..e82e39c7e 100644
--- a/Makefile
+++ b/Makefile
@@ -1,53 +1,51 @@
-#
-# usage:
-# make infest system=foo [target=bar]
-# make [deploy] system=foo [target=bar]
-# make [deploy] systems='foo bar'
-# make eval get=users.tv.wu.config.time.timeZone [filter=json]
-#
-
.ONESHELL:
.SHELLFLAGS := -eufc
-ifdef systems
-$(systems):
- @
- unset target
- parallel \
- --line-buffer \
- -j0 \
- --no-notice \
- --tagstring {} \
- -q make -s systems= system={} ::: $(systems)
-else ifdef system
-.PHONY: deploy infest
-deploy infest:;@
- export get=krebs.$@
- export filter=json
- make -s eval | sh
+ifndef system
+$(error unbound variable: system)
+endif
+
+export target_host ?= $(system)
+export target_user ?= root
+export target_path ?= /var/src
+
+# usage: make deploy system=foo [target_host=bar]
+.PHONY: deploy
+deploy: populate ;@set -x
+ ssh "$$target_user@$$target_host" nixos-rebuild switch -I "$$target_path"
+# usage: make populate system=foo [target_host=bar]
+.PHONY: populate
+populate:;@
+ result=$$(make -s eval get=config.krebs.build.populate filter=json)
+ echo "$$result" | sh
+
+# usage: make eval system=foo get=config.krebs.build [LOGNAME=tv] [filter=json]
.PHONY: eval
-eval:
- @
+eval:;@
ifeq ($(filter),json)
extraArgs='--json --strict'
- filter() { jq -r .; }
+ filter() { echo "$$1" | jq -r .; }
else
- filter() { cat; }
+ filter() { echo "$$1"; }
endif
result=$$(nix-instantiate \
$${extraArgs-} \
+ --show-trace \
+ --readonly-mode \
--eval \
-A "$$get" \
- -I stockholm="$$PWD" \
- '<stockholm>' \
- --argstr current-date "$$(date -Is)" \
- --argstr current-host-name "$$HOSTNAME" \
- --argstr current-user-name "$$LOGNAME" \
- $${system+--argstr system "$$system"} \
- $${target+--argstr target "$$target"})
- echo "$$result" | filter
+ --arg configuration "./$$LOGNAME/1systems/$$system.nix")
+ filter "$$result"
-else
-$(error unbound variable: system[s])
-endif
+## usage: make install system=foo target=
+#.PHONY: install
+#install: ssh = ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+#install:;@set -x
+# $(ssh) "$$target_user@$$target_host" \
+# env target_path=/var/src \
+# sh -s prepare < krebs/4lib/infest/prepare.sh
+# make -s populate target_path=/mnt"$$target_path"
+# $(ssh) "$$target_user@$$target_host" \
+# env NIXOS_CONFIG=/var/src/nixos-config \
+# nixos-install
diff --git a/default.nix b/default.nix
index 1637aa464..278f1d14d 100644
--- a/default.nix
+++ b/default.nix
@@ -1,6 +1,15 @@
-{ current-date ? abort "current-date not defined"
-, current-host-name ? abort "current-host-name not defined"
-, current-user-name ? builtins.getEnv "LOGNAME"
+{ configuration ? import (nixpkgs-path + "/nixos/lib/from-env.nix") "NIXOS_CONFIG" <nixos-config>
+, system ? builtins.currentSystem
+, current-host-name ?
+ let v = builtins.getEnv "HOSTNAME"; in
+ if v != "" then v else builtins.readFile /proc/sys/kernel/hostname
+, current-user-name ?
+ let v = builtins.getEnv "LOGNAME"; in
+ if v != "" then v else abort "undefined variable: LOGNAME"
+, nixpkgs-path ?
+ if (builtins.tryEval <nixpkgs/krebs>).success
+ then <upstream-nixpkgs>
+ else <nixpkgs>
, StrictHostKeyChecking ? "yes"
}@args:
@@ -8,26 +17,26 @@ let stockholm = {
inherit krebs;
inherit users;
inherit lib;
- inherit pkgs;
+ inherit config options pkgs;
+ system = config.system.build.toplevel;
};
krebs = import ./krebs (args // { inherit lib stockholm; });
lib = let
- nlib = import <nixpkgs/lib>;
+ nlib = import (slib.npath "lib");
klib = import (slib.kpath "4lib") { lib = nlib; };
slib = rec {
- stockholm-path = ./.;
- nspath = ns: p: stockholm-path + "/${ns}/${p}";
- kpath = nspath "krebs";
- upath = nspath current-user-name;
+ npath = p: nixpkgs-path + "/${p}";
+ kpath = p: ./. + "/krebs/${p}";
+ upath = p: ./. + "/${current-user-name}/${p}";
};
ulib = let p = slib.upath "4lib"; in
nlib.optionalAttrs (klib.dir.has-default-nix p)
(import p { lib = nlib // klib; });
in nlib // klib // slib // ulib // builtins;
- inherit (eval {}) pkgs;
+ inherit (eval configuration) config options pkgs;
base-module = { config, ... }: {
imports = builtins.filter lib.dir.has-default-nix (lib.concatLists [
@@ -45,7 +54,8 @@ let stockholm = {
in kpkgs // upkgs;
};
- eval = config: import <nixpkgs/nixos/lib/eval-config.nix> {
+ eval = config: import (lib.npath "nixos/lib/eval-config.nix") {
+ inherit system;
specialArgs = {
inherit lib;
};
diff --git a/krebs/3modules/backup.nix b/krebs/3modules/backup.nix
index 01bb16a2b..a1f335905 100644
--- a/krebs/3modules/backup.nix
+++ b/krebs/3modules/backup.nix
@@ -28,9 +28,17 @@ let
type = types.krebs.file-location;
};
startAt = mkOption {
- type = types.str;
+ default = "hourly";
+ type = types.str; # TODO systemd.time(7)'s calendar event
};
snapshots = mkOption {
+ default = {
+ hourly = { format = "%Y-%m-%dT%H"; retain = 4; };
+ daily = { format = "%Y-%m-%d"; retain = 7; };
+ weekly = { format = "%YW%W"; retain = 4; };
+ monthly = { format = "%Y-%m"; retain = 12; };
+ yearly = { format = "%Y"; };
+ };
type = types.attrsOf (types.submodule {
options = {
format = mkOption {
@@ -284,3 +292,15 @@ let
};
in out
+# TODO ionice
+# TODO mail on failed push, pull
+# TODO mail on missing push
+# TODO don't cancel plans on activation
+# also, don't hang while deploying at:
+# starting the following units: backup.wu-home-xu.push.service, backup.wu-home-xu.push.timer
+# TODO make sure /bku is properly mounted
+# TODO make sure that secure hosts cannot backup to insecure ones
+# TODO optionally only backup when src and dst are near enough :)
+# TODO try using btrfs for snapshots (configurable)
+# TODO warn if partial snapshots are found
+# TODO warn if unknown stuff is found in dst path
diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix
index 7f004cd81..00142acdd 100644
--- a/krebs/3modules/build.nix
+++ b/krebs/3modules/build.nix
@@ -28,48 +28,159 @@ let
type = types.user;
};
- options.krebs.build.source.dir = mkOption {
- type = let
- default-host = config.krebs.current.host;
- in types.attrsOf (types.submodule ({ config, ... }: {
+ options.krebs.build.source = let
+ raw = types.either types.str types.path;
+ url = types.submodule {
options = {
- host = mkOption {
- type = types.host;
- default = default-host;
- };
- path = mkOption {
- type = types.str;
- };
- target-path = mkOption {
- type = types.str;
- default = "/root/${config._module.args.name}";
- };
url = mkOption {
type = types.str;
- default = "file://${config.host.name}${config.path}";
- };
- };
- }));
- default = {};
- };
-
- options.krebs.build.source.git = mkOption {
- type = with types; attrsOf (submodule ({ config, ... }: {
- options = {
- url = mkOption {
- type = types.str; # TODO must be shell safe
};
rev = mkOption {
type = types.str;
};
- target-path = mkOption {
+ dev = mkOption {
type = types.str;
- default = "/root/${config._module.args.name}";
};
};
- }));
+ };
+ in mkOption {
+ type = types.attrsOf (types.either types.str url);
+ apply = let f = mapAttrs (_: value: {
+ string = value;
+ path = toString value;
+ set = f value;
+ }.${typeOf value}); in f;
default = {};
};
+
+ options.krebs.build.populate = mkOption {
+ type = types.str;
+ default = let
+ source = config.krebs.build.source;
+ target-user = maybeEnv "target_user" "root";
+ target-host = maybeEnv "target_host" config.krebs.build.host.name;
+ target-path = maybeEnv "target_path" "/var/src";
+ out = ''
+ #! /bin/sh
+ set -eu
+
+ verbose() {
+ printf '+%s\n' "$(printf ' %q' "$@")" >&2
+ "$@"
+ }
+
+ echo ${shell.escape git-script} \
+ | ssh ${shell.escape "${target-user}@${target-host}"} -T
+
+ unset tmpdir
+ trap '
+ rm "$tmpdir"/*
+ rmdir "$tmpdir"
+ trap - EXIT INT QUIT
+ ' EXIT INT QUIT
+ tmpdir=$(mktemp -dt stockholm.XXXXXXXX)
+ chmod 0755 "$tmpdir"
+
+ ${concatStringsSep "\n"
+ (mapAttrsToList
+ (name: spec: let dst = removePrefix "symlink:" (get-url spec); in
+ "verbose ln -s ${shell.escape dst} $tmpdir/${shell.escape name}")
+ symlink-specs)}
+
+ verbose proot \
+ -b $tmpdir:${shell.escape target-path} \
+ ${concatStringsSep " \\\n "
+ (mapAttrsToList
+ (name: spec:
+ "-b ${shell.escape "${get-url spec}:${target-path}/${name}"}")
+ file-specs)} \
+ rsync \
+ -f ${shell.escape "P /*"} \
+ ${concatMapStringsSep " \\\n "
+ (name: "-f ${shell.escape "R /${name}"}")
+ (attrNames file-specs)} \
+ --delete \
+ -vFrlptD \
+ ${shell.escape target-path}/ \
+ ${shell.escape "${target-user}@${target-host}:${target-path}"}
+ '';
+
+ get-schema = uri:
+ if substring 0 1 uri == "/"
+ then "file"
+ else head (splitString ":" uri);
+
+ has-schema = schema: uri: get-schema uri == schema;
+
+ get-url = spec: {
+ string = spec;
+ path = toString spec;
+ set = get-url spec.url;
+ }.${typeOf spec};
+
+ git-specs =
+ filterAttrs (_: spec: has-schema "https" (get-url spec)) source //
+ filterAttrs (_: spec: has-schema "http" (get-url spec)) source //
+ filterAttrs (_: spec: has-schema "git" (get-url spec)) source;
+
+ file-specs =
+ filterAttrs (_: spec: has-schema "file" (get-url spec)) source;
+
+ symlink-specs =
+ filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source;
+
+ git-script = ''
+ #! /bin/sh
+ set -efu
+
+ verbose() {
+ printf '+%s\n' "$(printf ' %q' "$@")" >&2
+ "$@"
+ }
+
+ fetch_git() {(
+ dst_dir=$1
+ src_url=$2
+ src_ref=$3
+
+ if ! test -e "$dst_dir"; then
+ git clone "$src_url" "$dst_dir"
+ fi
+
+ cd "$dst_dir"
+
+ if ! url=$(git config remote.origin.url); then
+ git remote add origin "$src_url"
+ elif test "$url" != "$src_url"; then
+ git remote set-url origin "$src_url"
+ fi
+
+ # TODO resolve src_ref to commit hash
+ hash=$src_ref
+
+ if ! test "$(git log --format=%H -1)" = "$hash"; then
+ git fetch origin
+ git checkout "$hash" -- "$dst_dir"
+ git checkout "$hash"
+ fi
+
+ git clean -dxf
+ )}
+
+ ${concatStringsSep "\n"
+ (mapAttrsToList
+ (name: spec: toString (map shell.escape [
+ "verbose"
+ "fetch_git"
+ "${target-path}/${name}"
+ spec.url
+ spec.rev
+ ]))
+ git-specs)}
+ '';
+ in out;
+ };
+
};
in out
diff --git a/krebs/3modules/git.nix b/krebs/3modules/git.nix
index e6267d7e6..7b28ffca8 100644
--- a/krebs/3modules/git.nix
+++ b/krebs/3modules/git.nix
@@ -27,7 +27,7 @@ let
description = ''
Enable cgit.
Cgit is an attempt to create a fast web interface for the git version
- control system, using a built in cache to decrease pressure on the
+ control system, using a built in cache to decrease pressure on the
git server.
cgit in this module is being served via fastcgi nginx.This module
deploys a http://cgit.<hostname> nginx configuration and enables nginx
@@ -44,48 +44,8 @@ let
default = "/etc/git";
};
repos = mkOption {
- type = types.attrsOf (types.submodule ({
- options = {
- desc = mkOption {
- type = types.nullOr types.str;
- default = null;
- description = ''
- Repository description.
- '';
- };
- section = mkOption {
- type = types.nullOr types.str;
- default = null;
- description = ''
- Repository section.
- '';
- };
- name = mkOption {
- type = types.str;
- description = ''
- Repository name.
- '';
- };
- hooks = mkOption {
- type = types.attrsOf types.str;
- default = {};
- description = ''
- Repository-specific hooks.
- '';
- };
- public = mkOption {
- type = types.bool;
- default = false;
- description = ''
- Allow everybody to read the repository via HTTP if cgit enabled.
- '';
- # TODO allow every configured user to fetch the repository via SSH.
- };
- };
- }));
-
+ type = types.attrsOf subtypes.repo;
default = {};
-
example = literalExample ''
{
testing = {
@@ -99,7 +59,6 @@ let
testing2 = { name = "testing2"; };
}
'';
-
description = ''
Repositories.
'';
@@ -121,30 +80,158 @@ let
'';
};
rules = mkOption {
- type = types.unspecified;
+ type = types.listOf subtypes.rule;
+ default = [];
+ example = literalExample ''
+ singleton {
+ user = [ config.krebs.users.tv ];
+ repo = [ testing ]; # see literal example of repos
+ perm = push "refs/*" (with lib.git; [
+ non-fast-forward create delete merge
+ ]);
+ }
+ '';
+ description = ''
+ Rules.
+ '';
};
};
+ # TODO put into krebs/4lib/types.nix?
+ subtypes = {
+ repo = types.submodule ({
+ options = {
+ collaborators = mkOption {
+ type = types.listOf types.user;
+ default = [];
+ description = ''
+ List of users that should be able to fetch from this repo.
+
+ This option is currently not used by krebs.git but instead can be
+ used to create rules. See e.g. <stockholm/tv/2configs/git.nix> for
+ an example.
+ '';
+ };
+ desc = mkOption {
+ type = types.nullOr types.str;
+ default = null;
+ description = ''
+ Repository description.
+ '';
+ };
+ section = mkOption {
+ type = types.nullOr types.str;
+ default = null;
+ description = ''
+ Repository section.
+ '';
+ };
+ name = mkOption