summaryrefslogtreecommitdiffstats
path: root/krebs/3modules
diff options
context:
space:
mode:
Diffstat (limited to 'krebs/3modules')
-rw-r--r--krebs/3modules/backup.nix22
-rw-r--r--krebs/3modules/build.nix171
-rw-r--r--krebs/3modules/git.nix201
-rw-r--r--krebs/3modules/tv/default.nix1
-rw-r--r--krebs/3modules/urlwatch.nix46
5 files changed, 351 insertions, 90 deletions
diff --git a/krebs/3modules/backup.nix b/krebs/3modules/backup.nix
index 01bb16a2b..a1f335905 100644
--- a/krebs/3modules/backup.nix
+++ b/krebs/3modules/backup.nix
@@ -28,9 +28,17 @@ let
type = types.krebs.file-location;
};
startAt = mkOption {
- type = types.str;
+ default = "hourly";
+ type = types.str; # TODO systemd.time(7)'s calendar event
};
snapshots = mkOption {
+ default = {
+ hourly = { format = "%Y-%m-%dT%H"; retain = 4; };
+ daily = { format = "%Y-%m-%d"; retain = 7; };
+ weekly = { format = "%YW%W"; retain = 4; };
+ monthly = { format = "%Y-%m"; retain = 12; };
+ yearly = { format = "%Y"; };
+ };
type = types.attrsOf (types.submodule {
options = {
format = mkOption {
@@ -284,3 +292,15 @@ let
};
in out
+# TODO ionice
+# TODO mail on failed push, pull
+# TODO mail on missing push
+# TODO don't cancel plans on activation
+# also, don't hang while deploying at:
+# starting the following units: backup.wu-home-xu.push.service, backup.wu-home-xu.push.timer
+# TODO make sure /bku is properly mounted
+# TODO make sure that secure hosts cannot backup to insecure ones
+# TODO optionally only backup when src and dst are near enough :)
+# TODO try using btrfs for snapshots (configurable)
+# TODO warn if partial snapshots are found
+# TODO warn if unknown stuff is found in dst path
diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix
index 7f004cd81..00142acdd 100644
--- a/krebs/3modules/build.nix
+++ b/krebs/3modules/build.nix
@@ -28,48 +28,159 @@ let
type = types.user;
};
- options.krebs.build.source.dir = mkOption {
- type = let
- default-host = config.krebs.current.host;
- in types.attrsOf (types.submodule ({ config, ... }: {
+ options.krebs.build.source = let
+ raw = types.either types.str types.path;
+ url = types.submodule {
options = {
- host = mkOption {
- type = types.host;
- default = default-host;
- };
- path = mkOption {
- type = types.str;
- };
- target-path = mkOption {
- type = types.str;
- default = "/root/${config._module.args.name}";
- };
url = mkOption {
type = types.str;
- default = "file://${config.host.name}${config.path}";
- };
- };
- }));
- default = {};
- };
-
- options.krebs.build.source.git = mkOption {
- type = with types; attrsOf (submodule ({ config, ... }: {
- options = {
- url = mkOption {
- type = types.str; # TODO must be shell safe
};
rev = mkOption {
type = types.str;
};
- target-path = mkOption {
+ dev = mkOption {
type = types.str;
- default = "/root/${config._module.args.name}";
};
};
- }));
+ };
+ in mkOption {
+ type = types.attrsOf (types.either types.str url);
+ apply = let f = mapAttrs (_: value: {
+ string = value;
+ path = toString value;
+ set = f value;
+ }.${typeOf value}); in f;
default = {};
};
+
+ options.krebs.build.populate = mkOption {
+ type = types.str;
+ default = let
+ source = config.krebs.build.source;
+ target-user = maybeEnv "target_user" "root";
+ target-host = maybeEnv "target_host" config.krebs.build.host.name;
+ target-path = maybeEnv "target_path" "/var/src";
+ out = ''
+ #! /bin/sh
+ set -eu
+
+ verbose() {
+ printf '+%s\n' "$(printf ' %q' "$@")" >&2
+ "$@"
+ }
+
+ echo ${shell.escape git-script} \
+ | ssh ${shell.escape "${target-user}@${target-host}"} -T
+
+ unset tmpdir
+ trap '
+ rm "$tmpdir"/*
+ rmdir "$tmpdir"
+ trap - EXIT INT QUIT
+ ' EXIT INT QUIT
+ tmpdir=$(mktemp -dt stockholm.XXXXXXXX)
+ chmod 0755 "$tmpdir"
+
+ ${concatStringsSep "\n"
+ (mapAttrsToList
+ (name: spec: let dst = removePrefix "symlink:" (get-url spec); in
+ "verbose ln -s ${shell.escape dst} $tmpdir/${shell.escape name}")
+ symlink-specs)}
+
+ verbose proot \
+ -b $tmpdir:${shell.escape target-path} \
+ ${concatStringsSep " \\\n "
+ (mapAttrsToList
+ (name: spec:
+ "-b ${shell.escape "${get-url spec}:${target-path}/${name}"}")
+ file-specs)} \
+ rsync \
+ -f ${shell.escape "P /*"} \
+ ${concatMapStringsSep " \\\n "
+ (name: "-f ${shell.escape "R /${name}"}")
+ (attrNames file-specs)} \
+ --delete \
+ -vFrlptD \
+ ${shell.escape target-path}/ \
+ ${shell.escape "${target-user}@${target-host}:${target-path}"}
+ '';
+
+ get-schema = uri:
+ if substring 0 1 uri == "/"
+ then "file"
+ else head (splitString ":" uri);
+
+ has-schema = schema: uri: get-schema uri == schema;
+
+ get-url = spec: {
+ string = spec;
+ path = toString spec;
+ set = get-url spec.url;
+ }.${typeOf spec};
+
+ git-specs =
+ filterAttrs (_: spec: has-schema "https" (get-url spec)) source //
+ filterAttrs (_: spec: has-schema "http" (get-url spec)) source //
+ filterAttrs (_: spec: has-schema "git" (get-url spec)) source;
+
+ file-specs =
+ filterAttrs (_: spec: has-schema "file" (get-url spec)) source;
+
+ symlink-specs =
+ filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source;
+
+ git-script = ''
+ #! /bin/sh
+ set -efu
+
+ verbose() {
+ printf '+%s\n' "$(printf ' %q' "$@")" >&2
+ "$@"
+ }
+
+ fetch_git() {(
+ dst_dir=$1
+ src_url=$2
+ src_ref=$3
+
+ if ! test -e "$dst_dir"; then
+ git clone "$src_url" "$dst_dir"
+ fi
+
+ cd "$dst_dir"
+
+ if ! url=$(git config remote.origin.url); then
+ git remote add origin "$src_url"
+ elif test "$url" != "$src_url"; then
+ git remote set-url origin "$src_url"
+ fi
+
+ # TODO resolve src_ref to commit hash
+ hash=$src_ref
+
+ if ! test "$(git log --format=%H -1)" = "$hash"; then
+ git fetch origin
+ git checkout "$hash" -- "$dst_dir"
+ git checkout "$hash"
+ fi
+
+ git clean -dxf
+ )}
+
+ ${concatStringsSep "\n"
+ (mapAttrsToList
+ (name: spec: toString (map shell.escape [
+ "verbose"
+ "fetch_git"
+ "${target-path}/${name}"
+ spec.url
+ spec.rev
+ ]))
+ git-specs)}
+ '';
+ in out;
+ };
+
};
in out
diff --git a/krebs/3modules/git.nix b/krebs/3modules/git.nix
index e6267d7e6..7b28ffca8 100644
--- a/krebs/3modules/git.nix
+++ b/krebs/3modules/git.nix
@@ -27,7 +27,7 @@ let
description = ''
Enable cgit.
Cgit is an attempt to create a fast web interface for the git version
- control system, using a built in cache to decrease pressure on the
+ control system, using a built in cache to decrease pressure on the
git server.
cgit in this module is being served via fastcgi nginx.This module
deploys a http://cgit.<hostname> nginx configuration and enables nginx
@@ -44,48 +44,8 @@ let
default = "/etc/git";
};
repos = mkOption {
- type = types.attrsOf (types.submodule ({
- options = {
- desc = mkOption {
- type = types.nullOr types.str;
- default = null;
- description = ''
- Repository description.
- '';
- };
- section = mkOption {
- type = types.nullOr types.str;
- default = null;
- description = ''
- Repository section.
- '';
- };
- name = mkOption {
- type = types.str;
- description = ''
- Repository name.
- '';
- };
- hooks = mkOption {
- type = types.attrsOf types.str;
- default = {};
- description = ''
- Repository-specific hooks.
- '';
- };
- public = mkOption {
- type = types.bool;
- default = false;
- description = ''
- Allow everybody to read the repository via HTTP if cgit enabled.
- '';
- # TODO allow every configured user to fetch the repository via SSH.
- };
- };
- }));
-
+ type = types.attrsOf subtypes.repo;
default = {};
-
example = literalExample ''
{
testing = {
@@ -99,7 +59,6 @@ let
testing2 = { name = "testing2"; };
}
'';
-
description = ''
Repositories.
'';
@@ -121,30 +80,158 @@ let
'';
};
rules = mkOption {
- type = types.unspecified;
+ type = types.listOf subtypes.rule;
+ default = [];
+ example = literalExample ''
+ singleton {
+ user = [ config.krebs.users.tv ];
+ repo = [ testing ]; # see literal example of repos
+ perm = push "refs/*" (with lib.git; [
+ non-fast-forward create delete merge
+ ]);
+ }
+ '';
+ description = ''
+ Rules.
+ '';
};
};
+ # TODO put into krebs/4lib/types.nix?
+ subtypes = {
+ repo = types.submodule ({
+ options = {
+ collaborators = mkOption {
+ type = types.listOf types.user;
+ default = [];
+ description = ''
+ List of users that should be able to fetch from this repo.
+
+ This option is currently not used by krebs.git but instead can be
+ used to create rules. See e.g. <stockholm/tv/2configs/git.nix> for
+ an example.
+ '';
+ };
+ desc = mkOption {
+ type = types.nullOr types.str;
+ default = null;
+ description = ''
+ Repository description.
+ '';
+ };
+ section = mkOption {
+ type = types.nullOr types.str;
+ default = null;
+ description = ''
+ Repository section.
+ '';
+ };
+ name = mkOption {
+ type = types.str;
+ description = ''
+ Repository name.
+ '';
+ };
+ hooks = mkOption {
+ type = types.attrsOf types.str;
+ default = {};
+ description = ''
+ Repository-specific hooks.
+ '';
+ };
+ public = mkOption {
+ type = types.bool;
+ default = false;
+ description = ''
+ Allow everybody to read the repository via HTTP if cgit enabled.
+ '';
+ # TODO allow every configured user to fetch the repository via SSH.
+ };
+ };
+ });
+ rule = types.submodule ({ config, ... }: {
+ options = {
+ user = mkOption {
+ type = types.listOf types.user;
+ description = ''
+ List of users this rule should apply to.
+ Checked by authorize-command.
+ '';
+ };
+ repo = mkOption {
+ type = types.listOf subtypes.repo;
+ description = ''
+ List of repos this rule should apply to.
+ Checked by authorize-command.
+ '';
+ };
+ perm = mkOption {
+ type = types.submodule {
+ # TODO generate enum argument from krebs/4lib/git.nix
+ options = {
+ allow-commands = mkOption {
+ type = types.listOf (types.enum (with git; [
+ git-receive-pack
+ git-upload-pack
+ ]));
+ default = [];
+ description = ''
+ List of commands the rule's users are allowed to execute.
+ Checked by authorize-command.
+ '';
+ };
+ allow-receive-ref = mkOption {
+ type = types.nullOr types.str;
+ default = null;
+ description = ''
+ Ref that can receive objects.
+ Checked by authorize-push.
+ '';
+ };
+ allow-receive-modes = mkOption {
+ type = types.listOf (types.enum (with git; [
+ fast-forward
+ non-fast-forward
+ create
+ delete
+ merge
+ ]));
+ default = [];
+ description = ''
+ List of allowed receive modes.
+ Checked by pre-receive hook.
+ '';
+ };
+ };
+ };
+ description = ''
+ Permissions granted.
+ '';
+ };
+ };
+ });
+ };
+
git-imp = {
system.activationScripts.git-init = "${init-script}";
-
+
# TODO maybe put all scripts here and then use PATH?
environment.etc."${etc-base}".source =
scriptFarm "git-ssh-authorizers" {
- authorize-command = makeAuthorizeScript (map ({ repo, user, perm }: [
- (map getName (ensureList user))
- (map getName (ensureList repo))
- (map getName perm.allow-commands)
+ authorize-command = makeAuthorizeScript (map (rule: [
+ (map getName (ensureList rule.user))
+ (map getName (ensureList rule.repo))
+ (map getName rule.perm.allow-commands)
]) cfg.rules);
-
- authorize-push = makeAuthorizeScript (map ({ repo, user, perm }: [
- (map getName (ensureList user))
- (map getName (ensureList repo))
- (ensureList perm.allow-receive-ref)
- (map getName perm.allow-receive-modes)
- ]) (filter (x: hasAttr "allow-receive-ref" x.perm) cfg.rules));
+
+ authorize-push = makeAuthorizeScript (map (rule: [
+ (map getName (ensureList rule.user))
+ (map getName (ensureList rule.repo))
+ (ensureList rule.perm.allow-receive-ref)
+ (map getName rule.perm.allow-receive-modes)
+ ]) (filter (rule: rule.perm.allow-receive-ref != null) cfg.rules));
};
-
+
users.extraUsers = singleton rec {
description = "Git repository hosting user";
name = "git";
diff --git a/krebs/3modules/tv/default.nix b/krebs/3modules/tv/default.nix
index 6fd1c4224..31c1a375a 100644
--- a/krebs/3modules/tv/default.nix
+++ b/krebs/3modules/tv/default.nix
@@ -247,6 +247,7 @@ with lib;
};
};
secure = true;
+ ssh.privkey.path = <secrets/ssh.id_ed25519>;
ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIcJvu8JDVzObLUtlAQg9qVugthKSfitwCljuJ5liyHa";
};
xu = {
diff --git a/krebs/3modules/urlwatch.nix b/krebs/3modules/urlwatch.nix
index 31cbfcf6e..7a9fb55fd 100644
--- a/krebs/3modules/urlwatch.nix
+++ b/krebs/3modules/urlwatch.nix
@@ -54,6 +54,10 @@ let
example = [
https://nixos.org/channels/nixos-unstable/git-revision
];
+ apply = map (x: getAttr (typeOf x) {
+ set = x;
+ string.url = x;
+ });
};
verbose = mkOption {
type = types.bool;
@@ -64,7 +68,40 @@ let
};
};
- urlsFile = toFile "urls" (concatStringsSep "\n" cfg.urls);
+ urlsFile = toFile "urls" (concatMapStringsSep "\n---\n" toJSON cfg.urls);
+
+ configFile = toFile "urlwatch.yaml" (toJSON {
+ display = {
+ error = true;
+ new = true;
+ unchanged = false;
+ };
+ report = {
+ email = {
+ enabled = false;
+ from = "";
+ html = false;
+ smtp = {
+ host = "localhost";
+ keyring = true;
+ port = 25;
+ starttls = true;
+ };
+ subject = "{count} changes: {jobs}";
+ to = "";
+ };
+ html.diff = "unified";
+ stdout = {
+ color = true;
+ enabled = true;
+ };
+ text = {
+ details = true;
+ footer = true;
+ line_length = 75;
+ };
+ };
+ });
imp = {
systemd.timers.urlwatch = {
@@ -109,10 +146,15 @@ let
from=${escapeShellArg cfg.from}
mailto=${escapeShellArg cfg.mailto}
urlsFile=${escapeShellArg urlsFile}
+ configFile=${escapeShellArg configFile}
cd /tmp
- urlwatch -e ${optionalString cfg.verbose "-v"} --urls="$urlsFile" > changes || :
+ urlwatch \
+ ${optionalString cfg.verbose "-v"} \
+ --urls="$urlsFile" \
+ --config="$configFile" \
+ > changes || :
if test -s changes; then
date=$(date -R)