diff options
-rw-r--r-- | .rsync-filter | 2 | ||||
-rw-r--r-- | Makefile | 22 | ||||
-rw-r--r-- | krebs/3modules/build.nix | 109 | ||||
-rw-r--r-- | krebs/3modules/urlwatch.nix | 46 | ||||
-rw-r--r-- | krebs/5pkgs/lentil/default.nix | 15 | ||||
-rw-r--r-- | krebs/5pkgs/lentil/syntaxes.patch | 11 | ||||
-rw-r--r-- | krebs/5pkgs/much/default.nix | 2 | ||||
-rw-r--r-- | krebs/5pkgs/urlwatch/default.nix | 39 | ||||
-rw-r--r-- | krebs/default.nix | 9 | ||||
-rw-r--r-- | krebs/populate.nix | 116 | ||||
-rw-r--r-- | nixpkgs/default.nix | 1 | ||||
-rw-r--r-- | nixpkgs/nixos/default.nix | 65 | ||||
l--------- | nixpkgs/nixos/modules | 1 | ||||
-rw-r--r-- | tv/1systems/cd.nix | 5 | ||||
-rw-r--r-- | tv/1systems/wu.nix | 2 | ||||
-rw-r--r-- | tv/1systems/xu.nix | 2 | ||||
-rw-r--r-- | tv/2configs/default.nix | 39 | ||||
-rw-r--r-- | tv/5pkgs/default.nix | 3 | ||||
-rw-r--r-- | tv/5pkgs/ejabberd/default.nix | 28 |
19 files changed, 425 insertions, 92 deletions
diff --git a/.rsync-filter b/.rsync-filter new file mode 100644 index 000000000..d7657cd00 --- /dev/null +++ b/.rsync-filter @@ -0,0 +1,2 @@ +- /.git +- /.graveyard @@ -24,7 +24,27 @@ else ifdef system deploy infest:;@ export get=krebs.$@ export filter=json - make -s eval | sh + script=$$(make -s eval) + echo "$$script" | sh + +.PHONY: deploy2 +ifdef target +deploy2: export target-host = $(target) +else +deploy2: export target-host = $(system) +endif +deploy2:;@ + target=$${target-$$system} + result=$$(nix-instantiate \ + --json \ + --eval \ + krebs/populate.nix \ + --arg source 'with (import ~/stockholm {}).users.$(LOGNAME).$(system).config.krebs.build; assert source-version == 2; source' \ + --argstr target-host "$$target" \ + --argstr target-path /var/src) + script=$$(echo "$$result" | jq -r .) + echo "$$script" | sh + ssh root@$$target nixos-rebuild switch -I /var/src .PHONY: eval eval: diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix index 7f004cd81..0f8aec89d 100644 --- a/krebs/3modules/build.nix +++ b/krebs/3modules/build.nix @@ -28,48 +28,83 @@ let type = types.user; }; - options.krebs.build.source.dir = mkOption { - type = let - default-host = config.krebs.current.host; - in types.attrsOf (types.submodule ({ config, ... }: { - options = { - host = mkOption { - type = types.host; - default = default-host; - }; - path = mkOption { - type = types.str; - }; - target-path = mkOption { - type = types.str; - default = "/root/${config._module.args.name}"; - }; - url = mkOption { - type = types.str; - default = "file://${config.host.name}${config.path}"; - }; - }; - })); - default = {}; + options.krebs.build.source-version = mkOption { + type = types.enum [ 1 2 ]; + default = 1; }; - options.krebs.build.source.git = mkOption { - type = with types; attrsOf (submodule ({ config, ... }: { - options = { - url = mkOption { - type = types.str; # TODO must be shell safe - }; - rev = mkOption { - type = types.str; - }; - target-path = mkOption { - type = types.str; - default = "/root/${config._module.args.name}"; + options.krebs.build.source = getAttr "v${toString config.krebs.build.source-version}" { + v1 = { + dir = mkOption { + type = let + default-host = config.krebs.current.host; + in types.attrsOf (types.submodule ({ config, ... }: { + options = { + host = mkOption { + type = types.host; + default = default-host; + }; + path = mkOption { + type = types.str; + }; + target-path = mkOption { + type = types.str; + default = "/root/${config._module.args.name}"; + }; + url = mkOption { + type = types.str; + default = "file://${config.host.name}${config.path}"; + }; + }; + })); + default = {}; + }; + + git = mkOption { + type = with types; attrsOf (submodule ({ config, ... }: { + options = { + url = mkOption { + type = types.str; # TODO must be shell safe + }; + rev = mkOption { + type = types.str; + }; + target-path = mkOption { + type = types.str; + default = "/root/${config._module.args.name}"; + }; + }; + })); + default = {}; + }; + }; + + v2 = let + raw = types.either types.str types.path; + url = types.submodule { + options = { + url = mkOption { + type = types.str; + }; + rev = mkOption { + type = types.str; + }; + dev = mkOption { + type = types.str; + }; }; }; - })); - default = {}; + in mkOption { + type = types.attrsOf (types.either types.str url); + apply = let f = mapAttrs (_: value: { + string = value; + path = toString value; + set = f value; + }.${typeOf value}); in f; + default = {}; + }; }; + }; in out diff --git a/krebs/3modules/urlwatch.nix b/krebs/3modules/urlwatch.nix index 31cbfcf6e..7a9fb55fd 100644 --- a/krebs/3modules/urlwatch.nix +++ b/krebs/3modules/urlwatch.nix @@ -54,6 +54,10 @@ let example = [ https://nixos.org/channels/nixos-unstable/git-revision ]; + apply = map (x: getAttr (typeOf x) { + set = x; + string.url = x; + }); }; verbose = mkOption { type = types.bool; @@ -64,7 +68,40 @@ let }; }; - urlsFile = toFile "urls" (concatStringsSep "\n" cfg.urls); + urlsFile = toFile "urls" (concatMapStringsSep "\n---\n" toJSON cfg.urls); + + configFile = toFile "urlwatch.yaml" (toJSON { + display = { + error = true; + new = true; + unchanged = false; + }; + report = { + email = { + enabled = false; + from = ""; + html = false; + smtp = { + host = "localhost"; + keyring = true; + port = 25; + starttls = true; + }; + subject = "{count} changes: {jobs}"; + to = ""; + }; + html.diff = "unified"; + stdout = { + color = true; + enabled = true; + }; + text = { + details = true; + footer = true; + line_length = 75; + }; + }; + }); imp = { systemd.timers.urlwatch = { @@ -109,10 +146,15 @@ let from=${escapeShellArg cfg.from} mailto=${escapeShellArg cfg.mailto} urlsFile=${escapeShellArg urlsFile} + configFile=${escapeShellArg configFile} cd /tmp - urlwatch -e ${optionalString cfg.verbose "-v"} --urls="$urlsFile" > changes || : + urlwatch \ + ${optionalString cfg.verbose "-v"} \ + --urls="$urlsFile" \ + --config="$configFile" \ + > changes || : if test -s changes; then date=$(date -R) diff --git a/krebs/5pkgs/lentil/default.nix b/krebs/5pkgs/lentil/default.nix deleted file mode 100644 index fc9b4fd31..000000000 --- a/krebs/5pkgs/lentil/default.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ pkgs, ... }: - -(pkgs.haskellngPackages.override { - overrides = self: super: { - lentil = super.lentil.override { - mkDerivation = (attrs: self.mkDerivation (attrs // { - version = "0.1.3.0"; - sha256 = "0xa59avh0bvfg69xh9p5b8dppfhx29mvfq8v41sk9j7qbcnzjivg"; - patches = [ - ./syntaxes.patch - ]; - })); - }; - }; -}).lentil diff --git a/krebs/5pkgs/lentil/syntaxes.patch b/krebs/5pkgs/lentil/syntaxes.patch deleted file mode 100644 index a9390ae51..000000000 --- a/krebs/5pkgs/lentil/syntaxes.patch +++ /dev/null @@ -1,11 +0,0 @@ -diff -rN -u old-lentil/src/Lentil/Parse/Syntaxes.hs new-lentil/src/Lentil/Parse/Syntaxes.hs ---- old-lentil/src/Lentil/Parse/Syntaxes.hs 2015-07-20 23:15:38.600539779 +0200 -+++ new-lentil/src/Lentil/Parse/Syntaxes.hs 2015-07-20 23:15:38.600539779 +0200 -@@ -30,6 +30,7 @@ - | ext `elem` [".pas", ".pp", ".inc"] = Just pascal - | ext `elem` [".py"] = Just python - | ext `elem` [".rb"] = Just ruby -+ | ext `elem` [".nix"] = Just perl -- Nix - | ext `elem` [".pl", ".pm", ".t"] = Just perl - | ext `elem` [".sh"] = Just perl -- shell - | ext `elem` [".txt"] = Just text diff --git a/krebs/5pkgs/much/default.nix b/krebs/5pkgs/much/default.nix index 2a9376139..efe84fbf0 100644 --- a/krebs/5pkgs/much/default.nix +++ b/krebs/5pkgs/much/default.nix @@ -1,6 +1,6 @@ { pkgs, ... }: -pkgs.haskellngPackages.callPackage ( +pkgs.haskellPackages.callPackage ( { mkDerivation, aeson, attoparsec, base, base64-bytestring , blaze-builder, blessings, bytestring, case-insensitive, containers, deepseq , directory, docopt, email-header, fetchgit, filepath diff --git a/krebs/5pkgs/urlwatch/default.nix b/krebs/5pkgs/urlwatch/default.nix new file mode 100644 index 000000000..d9b595314 --- /dev/null +++ b/krebs/5pkgs/urlwatch/default.nix @@ -0,0 +1,39 @@ +{ stdenv, fetchurl, python3Packages }: + +python3Packages.buildPythonPackage rec { + name = "urlwatch-2.0"; + + src = fetchurl { + url = "https://thp.io/2008/urlwatch/${name}.tar.gz"; + sha256 = "0j38qzw4jxw41vnnpi6j851hqpv8d6p1cbni6cv8r2vqf5307s3b"; + }; + + propagatedBuildInputs = with python3Packages; [ + pyyaml + keyring + (python3Packages.buildPythonPackage rec { + name = "minidb-2.0.1"; + src = fetchurl { + url = "https://thp.io/2010/minidb/${name}.tar.gz"; + sha256 = "1x958zr9jc26vaqij451qb9m2l7apcpz34ir9fwfjg4fwv24z2dy"; + }; + meta = { + description = "A simple SQLite3-based store for Python objects"; + homepage = https://thp.io/2010/minidb/; + license = stdenv.lib.licenses.isc; + maintainers = [ stdenv.lib.maintainers.tv ]; + }; + }) + ]; + + postFixup = '' + wrapProgram "$out/bin/urlwatch" --prefix "PYTHONPATH" : "$PYTHONPATH" + ''; + + meta = { + description = "A tool for monitoring webpages for updates"; + homepage = https://thp.io/2008/urlwatch/; + license = stdenv.lib.licenses.bsd3; + maintainers = [ stdenv.lib.maintainers.tv ]; + }; +}# diff --git a/krebs/default.nix b/krebs/default.nix index 15d0e8e2e..df2d95483 100644 --- a/krebs/default.nix +++ b/krebs/default.nix @@ -171,9 +171,12 @@ let out = { ${b} ''; - get-config = system: - stockholm.users.${current-user-name}.${system}.config + get-config = system: let + config = stockholm.users.${current-user-name}.${system}.config or (abort "unknown system: ${system}, user: ${current-user-name}"); + in + assert config.krebs.build.source-version == 1; + config; nix-install = { system ? current-host-name @@ -240,7 +243,7 @@ let out = { --exclude tmp \ --rsync-path='mkdir -p ${target-path} && rsync' \ --delete-excluded \ - -vrLptgoD \ + -vrlptD \ ${config.path}/ \ root@${target}:${target-path} ''; diff --git a/krebs/populate.nix b/krebs/populate.nix new file mode 100644 index 000000000..13270c8a7 --- /dev/null +++ b/krebs/populate.nix @@ -0,0 +1,116 @@ +{ source +, target-user ? "root" +, target-host +, target-path ? "/var/src" +}: +with import <nixpkgs/lib>; +with import ~/stockholm/krebs/4lib { + lib = import <nixpkgs/lib>; +}; +with builtins; +let + out = '' + #! /bin/sh + set -efu + + echo ${shell.escape git-script} \ + | ssh ${shell.escape "${target-user}@${target-host}"} -T + + tmpdir=$(mktemp -dt stockholm.XXXXXXXX) + trap ' + set +f + rm "$tmpdir"/* + rmdir "$tmpdir" + trap - EXIT INT QUIT + ' EXIT INT QUIT + chmod 0755 "$tmpdir" + ${concatStringsSep "\n" + (mapAttrsToList + (name: spec: let dst = removePrefix "symlink:" (get-url spec); in + "ln -s ${shell.escape dst} $tmpdir/${shell.escape name}") + symlink-specs)} + + proot \ + -b $tmpdir:${shell.escape target-path} \ + ${concatStringsSep " \\\n " + (mapAttrsToList + (name: spec: + "-b ${shell.escape "${get-url spec}:${target-path}/${name}"}") + file-specs)} \ + rsync \ + -f ${shell.escape "P /*"} \ + ${concatMapStringsSep " \\\n " + (name: "-f ${shell.escape "R /${name}"}") + (attrNames file-specs)} \ + --delete \ + -vFrlptD \ + ${shell.escape target-path}/ \ + ${shell.escape "${target-user}@${target-host}:${target-path}"} + ''; + + get-schema = uri: + if substring 0 1 uri == "/" + then "file" + else head (splitString ":" uri); + + has-schema = schema: uri: get-schema uri == schema; + + get-url = spec: { + string = spec; + path = toString spec; + set = get-url spec.url; + }.${typeOf spec}; + + git-specs = + filterAttrs (_: spec: has-schema "https" (get-url spec)) source // + filterAttrs (_: spec: has-schema "http" (get-url spec)) source // + filterAttrs (_: spec: has-schema "git" (get-url spec)) source; + + file-specs = + filterAttrs (_: spec: has-schema "file" (get-url spec)) source; + + symlink-specs = + filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source; + + git-script = '' + fetch_git() {( + dst_dir=$1 + src_url=$2 + src_ref=$3 + + if ! test -e "$dst_dir"; then + git clone "$src_url" "$dst_dir" + fi + + cd "$dst_dir" + + if ! url=$(git config remote.origin.url); then + git remote add origin "$src_url" + elif test "$url" != "$src_url"; then + git remote set-url origin "$src_url" + fi + + # TODO resolve src_ref to commit hash + hash=$src_ref + + if ! test "$(git log --format=%H -1)" = "$hash"; then + git fetch origin + git checkout "$hash" -- "$dst_dir" + git checkout "$hash" + fi + + git clean -dxf + )} + + ${concatStringsSep "\n" + (mapAttrsToList + (name: spec: toString (map shell.escape [ + "fetch_git" + "${target-path}/${name}" + spec.url + spec.rev + ])) + git-specs)} + ''; + +in out diff --git a/nixpkgs/default.nix b/nixpkgs/default.nix new file mode 100644 index 000000000..92da82c35 --- /dev/null +++ b/nixpkgs/default.nix @@ -0,0 +1 @@ +import <upstream-nixpkgs> diff --git a/nixpkgs/nixos/default.nix b/nixpkgs/nixos/default.nix new file mode 100644 index 000000000..6c5adf365 --- /dev/null +++ b/nixpkgs/nixos/default.nix @@ -0,0 +1,65 @@ +{ configuration ? import <upstream-nixpkgs/nixos/lib/from-env.nix> "NIXOS_CONFIG" <nixos-config> +, system ? builtins.currentSystem +}: + +let + eval-config = modules: import <upstream-nixpkgs/nixos/lib/eval-config.nix> { + inherit system; + modules = modules ++ [({ config, lib, ... }: with lib; { + imports = filter dir.has-default-nix (concatLists [ + (map (p: p + "/2configs") [ <stockholm-private> ]) + (map (p: p + "/3modules") [ <stockholm-krebs> <stockholm-private> ]) + ]); + + krebs.current = { + enable = true; + host = config.krebs.hosts.${readFile /proc/sys/kernel/hostname}; + user = config.krebs.users.${getEnv "LOGNAME"}; + }; + + nixpkgs.config.packageOverrides = pkgs: let + kpkgs = import <stockholm-krebs/5pkgs> { inherit lib pkgs; }; + upkgs = import <stockholm-private/5pkgs> { inherit lib; pkgs = pkgs // kpkgs; }; + in kpkgs // upkgs; + })]; + specialArgs = { + lib = let + nlib = import <upstream-nixpkgs/lib> // builtins; + klib = nlib // import <stockholm-krebs/4lib> { lib = nlib; }; + ulib = klib // (with klib; let p = <stockholm-private> + "/4lib"; in + optionalAttrs (dir.has-default-nix p) + (import p { lib = klib; })); + in ulib; + }; + }; + + eval = eval-config [ + configuration + ]; + + # This is for `nixos-rebuild build-vm'. + vm = eval-config [ + configuration + <upstream-nixpkgs/nixos/modules/virtualisation/qemu-vm.nix> + ]; + + # This is for `nixos-rebuild build-vm-with-bootloader'. + vm-with-bootloader = eval-config [ + configuration + <upstream-nixpkgs/nixos/modules/virtualisation/qemu-vm.nix> + { virtualisation.useBootLoader = true; } + ]; +in + +{ + inherit (eval) config options; + + system = eval.config.system.build.toplevel; + + vm = vm.config.system.build.vm; + + vmWithBootLoader = vm-with-bootloader.config.system.build.vm; + + # The following are used by nixos-rebuild. + nixFallback = eval.pkgs.nixUnstable; +} diff --git a/nixpkgs/nixos/modules b/nixpkgs/nixos/modules new file mode 120000 index 000000000..8fbc4373e --- /dev/null +++ b/nixpkgs/nixos/modules @@ -0,0 +1 @@ +../../upstream-nixpkgs/nixos/modules
\ No newline at end of file diff --git a/tv/1systems/cd.nix b/tv/1systems/cd.nix index 8c2a9ae43..b69d7655a 100644 --- a/tv/1systems/cd.nix +++ b/tv/1systems/cd.nix @@ -6,6 +6,11 @@ with lib; krebs.build.host = config.krebs.hosts.cd; krebs.build.target = "root@cd.internet"; + krebs.build.source.upstream-nixpkgs = { + url = https://github.com/NixOS/nixpkgs; + rev = "b7ff030"; + }; + imports = [ ../2configs/hw/CAC-Developer-2.nix ../2configs/fs/CAC-CentOS-7-64bit.nix diff --git a/tv/1systems/wu.nix b/tv/1systems/wu.nix index 54ceb7783..3bdf8d37a 100644 --- a/tv/1systems/wu.nix +++ b/tv/1systems/wu.nix @@ -18,7 +18,7 @@ with lib; # stockholm gnumake hashPassword - lentil + haskellPackages.lentil parallel (pkgs.writeScriptBin "im" '' #! ${pkgs.bash}/bin/bash diff --git a/tv/1systems/xu.nix b/tv/1systems/xu.nix index 1f3e010a4..e6894b55e 100644 --- a/tv/1systems/xu.nix +++ b/tv/1systems/xu.nix @@ -21,7 +21,7 @@ with lib; # stockholm gnumake hashPassword - lentil + haskellPackages.lentil parallel (pkgs.writeScriptBin "im" '' #! ${pkgs.bash}/bin/bash diff --git a/tv/2configs/default.nix b/tv/2configs/default.nix index d3a1778f0..310077021 100644 --- a/tv/2configs/default.nix +++ b/tv/2configs/default.nix @@ -8,20 +8,23 @@ with lib; krebs.build = { user = config.krebs.users.tv; target = mkDefault "root@${config.krebs.build.host.name}"; - source = { - git.nixpkgs = { - url = mkDefault https://github.com/NixOS/nixpkgs; - rev = mkDefault "b7ff0301d6f26bd8419e888fd0e129f3dc8bd328"; - target-path = mkDefault "/var/src/nixpkgs"; + source-version = 2; + source = mapAttrs (_: mkDefault) ({ + nixos-config = "symlink:stockholm-private/1systems/${config.krebs.build.host.name}.nix"; + nixpkgs = symlink:stockholm-nixpkgs; + secrets = "/home/tv/secrets/${config.krebs.build.host.name}"; + secrets-common = "/home/tv/secrets/common"; + stockholm-krebs = "/home/tv/stockholm/krebs"; + stockholm-nixpkgs = "/home/tv/stockholm/nixpkgs"; + stockholm-private = "/home/tv/stockholm/tv"; + upstream-nixpkgs = { + url = https://github.com/NixOS/nixpkgs; + rev = "77f8f35d57618c1ba456d968524f2fb2c3448295"; + dev = "/home/tv/nixpkgs"; }; - dir.secrets = { - path = mkDefault "/home/tv/secrets/${config.krebs.build.host.name}"; - }; - dir.stockholm = { - path = mkDefault "/home/tv/stockholm"; - target-path = mkDefault "/var/src/stockholm"; - }; - }; + } // optionalAttrs config.krebs.build.host.secure { + secrets-master = "/home/tv/secrets/master"; + }); }; networking.hostName = config.krebs.build.host.name; @@ -98,12 +101,7 @@ with lib; }; environment.variables = { - NIX_PATH = - with config.krebs.build.source; with dir; with git; - mkForce (concatStringsSep ":" [ - "nixpkgs=${nixpkgs.target-path}" - "secrets=${stockholm.target-path}/null" - ]); + NIX_PATH = mkForce "/var/src"; }; programs.bash = { @@ -182,7 +180,8 @@ with lib; { systemd.tmpfiles.rules = let forUsers = flip map users; - isUser = { group, ... }: hasSuffix "users" group; + isUser = { name, group, ... }: + name == "root" || hasSuffix "users" group; users = filter isUser (mapAttrsToList (_: id) config.users.users); in forUsers (u: "d /run/xdg/${u.name} 0700 ${u.name} ${u.group} -"); environment.variables.XDG_RUNTIME_DIR = "/run/xdg/$LOGNAME"; diff --git a/tv/5pkgs/default.nix b/tv/5pkgs/default.nix index 2108d972c..be10e91eb 100644 --- a/tv/5pkgs/default.nix +++ b/tv/5pkgs/default.nix @@ -11,6 +11,9 @@ --disk-cache-size=50000000 \ "%@" ''; + ejabberd = pkgs.callPackage ./ejabberd { + erlang = pkgs.erlangR16; + }; ff = pkgs.callPackage ./ff {}; viljetic-pages = pkgs.callPackage ./viljetic-pages {}; xmonad-tv = diff --git a/tv/5pkgs/ejabberd/default.nix b/tv/5pkgs/ejabberd/default.nix new file mode 100644 index 000000000..3a77c5cd1 --- /dev/null +++ b/tv/5pkgs/ejabberd/default.nix @@ -0,0 +1,28 @@ +{stdenv, fetchurl, expat, erlang, zlib, openssl, pam, lib}: + +stdenv.mkDerivation rec { + version = "2.1.13"; + name = "ejabberd-${version}"; + src = fetchurl { + url = "http://www.process-one.net/downloads/ejabberd/${version}/${name}.tgz"; + sha256 = "0vf8mfrx7vr3c5h3nfp3qcgwf2kmzq20rjv1h9sk3nimwir1q3d8"; + }; + buildInputs = [ expat erlang zlib openssl pam ]; + patchPhase = '' + sed -i \ + -e "s|erl \\\|${erlang}/bin/erl \\\|" \ + -e 's|EXEC_CMD=\"sh -c\"|EXEC_CMD=\"${stdenv.shell} -c\"|' \ + src/ejabberdctl.template + ''; + preConfigure = '' + cd src + ''; + configureFlags = ["--enable-pam"]; + + meta = { + description = "Open-source XMPP application server written in Erlang"; + license = stdenv.lib.licenses.gpl2; + homepage = http://www.ejabberd.im; + maintainers = [ lib.maintainers.sander ]; + }; +} |