diff options
| -rw-r--r-- | Makefile | 23 | ||||
| -rw-r--r-- | krebs/3modules/build.nix | 138 | ||||
| -rw-r--r-- | krebs/4lib/types.nix | 69 | ||||
| -rwxr-xr-x | krebs/5pkgs/test/infest-cac-centos7/notes | 4 | ||||
| -rw-r--r-- | lass/2configs/buildbot-standalone.nix | 7 | ||||
| -rw-r--r-- | lass/2configs/default.nix | 18 | ||||
| -rw-r--r-- | makefu/2configs/default.nix | 20 | ||||
| -rw-r--r-- | shared/2configs/base.nix | 13 | ||||
| -rw-r--r-- | shared/2configs/shared-buildbot.nix | 8 | ||||
| -rw-r--r-- | tv/2configs/default.nix | 20 |
10 files changed, 127 insertions, 193 deletions
@@ -68,23 +68,27 @@ evaluate = \ -I stockholm=$(stockholm) \ -E "let eval = import <stockholm>; in with eval; $(1)" -execute = \ - result=$$($(call evaluate,config.krebs.build.$(1))) && \ - script=$$(echo "$$result" | jq -r .) && \ - echo "$$script" | PS5=% sh - ifeq ($(MAKECMDGOALS),) $(error No goals specified) endif # usage: make deploy system=foo [target_host=bar] deploy: ssh ?= ssh -deploy: - $(call execute,populate) +deploy: populate $(ssh) $(target_user)@$(target_host) -p $(target_port) \ env STOCKHOLM_VERSION="$$STOCKHOLM_VERSION" \ nixos-rebuild switch --show-trace -I $(target_path) +# usage: make populate system=foo +ifeq ($(debug),true) +populate: populate-flags = --debug +endif +populate: + source=$$($(call evaluate,config.krebs.build.source) --json --strict) && \ + echo $$source | populate \ + $(target_user)@$(target_host):$(target_port)$(target_path) \ + $(populate-flags) + # usage: make build.pkgs.get build build.:;@$(call build,$${expr-eval}) build.%:;@$(call build,$@) @@ -99,7 +103,7 @@ install: $(ssh) $(target_user)@$(target_host) -p $(target_port) \ env target_path=$(target_path) \ sh -s prepare < krebs/4lib/infest/prepare.sh - target_path=/mnt$(target_path) $(call execute,populate) + $(MAKE) populate target_path=/mnt$(target_path) $(ssh) $(target_user)@$(target_host) -p $(target_port) \ env NIXOS_CONFIG=$(target_path)/nixos-config \ STOCKHOLM_VERSION="$$STOCKHOLM_VERSION" \ @@ -117,8 +121,7 @@ $(error bad method: $(method)) endif endif test: ssh ?= ssh -test: - $(call execute,populate) +test: populate $(ssh) $(target_user)@$(target_host) -p $(target_port) \ $(command) --show-trace -I $(target_path) \ -A config.system.build.toplevel $(target_path)/stockholm diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix index 9cd095622..5924d1033 100644 --- a/krebs/3modules/build.nix +++ b/krebs/3modules/build.nix @@ -21,145 +21,9 @@ let }; options.krebs.build.source = mkOption { - type = with types; attrsOf (either str (submodule { - options = { - url = str; - rev = str; - }; - })); + type = types.attrsOf types.source; default = {}; }; - - options.krebs.build.populate = mkOption { - type = types.str; - default = let - target-user = maybeEnv "target_user" "root"; - target-host = maybeEnv "target_host" config.krebs.build.host.name; - target-port = maybeEnv "target_port" "22"; - target-path = maybeEnv "target_path" "/var/src"; - out = '' - #! /bin/sh - set -eu - - ssh=''${ssh-ssh} - - verbose() { - printf '%s%s\n' "$PS5$(printf ' %q' "$@")" >&2 - "$@" - } - - { printf 'PS5=%q%q\n' @ "$PS5" - echo ${shell.escape git-script} - } | verbose $ssh -p ${shell.escape target-port} \ - ${shell.escape "${target-user}@${target-host}"} -T - - unset tmpdir - trap ' - rm -f "$tmpdir"/* - rmdir "$tmpdir" - trap - EXIT INT QUIT - ' EXIT INT QUIT - tmpdir=$(mktemp -dt stockholm.XXXXXXXX) - chmod 0755 "$tmpdir" - - ${concatStringsSep "\n" (mapAttrsToList (name: symlink: '' - verbose ln -s ${shell.escape symlink.target} \ - "$tmpdir"/${shell.escape name} - '') source-by-method.symlink)} - - verbose proot \ - -b "$tmpdir":${shell.escape target-path} \ - ${concatStringsSep " \\\n " (mapAttrsToList (name: file: - "-b ${shell.escape "${file.path}:${target-path}/${name}"}" - ) source-by-method.file)} \ - rsync \ - -f ${shell.escape "P /*"} \ - ${concatMapStringsSep " \\\n " (name: - "-f ${shell.escape "R /${name}"}" - ) (attrNames source-by-method.file)} \ - --delete \ - -vFrlptD \ - -e "$ssh -p ${shell.escape target-port}" \ - ${shell.escape target-path}/ \ - ${shell.escape "${target-user}@${target-host}:${target-path}"} - ''; - - git-script = '' - #! /bin/sh - set -efu - - export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt - - verbose() { - printf '%s%s\n' "$PS5$(printf ' %q' "$@")" >&2 - "$@" - } - - fetch_git() {( - dst_dir=$1 - src_url=$2 - src_ref=$3 - - if ! test -e "$dst_dir"; then - git clone "$src_url" "$dst_dir" - fi - - cd "$dst_dir" - - if ! url=$(git config remote.origin.url); then - git remote add origin "$src_url" - elif test "$url" != "$src_url"; then - git remote set-url origin "$src_url" - fi - - # TODO resolve src_ref to commit hash - hash=$src_ref - - if ! test "$(git log --format=%H -1)" = "$hash"; then - git fetch origin - git checkout "$hash" -- "$dst_dir" - git checkout -f "$hash" - fi - - git clean -dxf - )} - - ${concatStringsSep "\n" (mapAttrsToList (name: git: '' - verbose fetch_git ${concatMapStringsSep " " shell.escape [ - "${target-path}/${name}" - git.url - git.rev - ]} - '') source-by-method.git)} - ''; - in out; - }; - - }; - - source-by-method = let - known-methods = ["git" "file" "symlink"]; - in genAttrs known-methods (const {}) // recursiveUpdate source-by-scheme { - git = source-by-scheme.http or {} // - source-by-scheme.https or {}; }; - source-by-scheme = foldl' (out: { k, v }: recursiveUpdate out { - ${v.scheme}.${k} = v; - }) {} (mapAttrsToList (k: v: { inherit k v; }) normalized-source); - - normalized-source = mapAttrs (name: let f = x: getAttr (typeOf x) { - path = f (toString x); - string = f { - url = if substring 0 1 x == "/" then "file://${x}" else x; - }; - set = let scheme = head (splitString ":" x.url); in recursiveUpdate x { - inherit scheme; - } // { - symlink.target = removePrefix "symlink:" x.url; - file.path = # TODO file://host/... - assert hasPrefix "file:///" x.url; - removePrefix "file://" x.url; - }.${scheme} or {}; - }; in f) config.krebs.build.source; in out diff --git a/krebs/4lib/types.nix b/krebs/4lib/types.nix index aa7b7a9f5..8906eff4a 100644 --- a/krebs/4lib/types.nix +++ b/krebs/4lib/types.nix @@ -188,6 +188,75 @@ types // rec { }; }); + + source = submodule ({ config, ... }: { + options = { + type = let + types = ["file" "git" "symlink"]; + in mkOption { + type = enum types; + default = let + cands = filter (k: config.${k} != null) types; + in + if length cands == 1 + then head cands + else throw "cannot determine type"; + }; + file = let + file-path = (file-source.getSubOptions "FIXME").path.type; + in mkOption { + type = nullOr (either file-source file-path); + default = null; + apply = x: + if file-path.check x + then { path = x; } + else x; + }; + git = mkOption { + type = nullOr git-source; + default = null; + }; + symlink = let + symlink-target = (symlink-source.getSubOptions "FIXME").target.type; + in mkOption { + type = nullOr (either symlink-source symlink-target); + default = null; + apply = x: + if symlink-target.check x + then { target = x; } + else x; + }; + }; + }); + + file-source = submodule { + options = { + path = mkOption { + type = absolute-pathname; + }; + }; + }; + + git-source = submodule { + options = { + ref = mkOption { + type = str; # TODO types.git.ref + }; + url = mkOption { + type = str; # TODO types.git.url + }; + }; + }; + + symlink-source = submodule { + options = { + target = mkOption { + type = pathname; # TODO relative-pathname + }; + }; + }; + + suffixed-str = suffs: mkOptionType { name = "string suffixed by ${concatStringsSep ", " suffs}"; diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes index ab6bc557c..2a3ebd6fc 100755 --- a/krebs/5pkgs/test/infest-cac-centos7/notes +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -138,8 +138,8 @@ ip=$(cac-api getserver $id | jq -r .ip) cat > shared/2configs/temp/dirs.nix <<EOF _: { krebs.build.source = { - secrets = "$krebs_secrets"; - stockholm = "$(pwd)"; + secrets.file = "$krebs_secrets"; + stockholm.file = "$(pwd)"; }; users.extraUsers.root.openssh.authorizedKeys.keys = [ "$(cat ${krebs_ssh}.pub)" diff --git a/lass/2configs/buildbot-standalone.nix b/lass/2configs/buildbot-standalone.nix index 04bdcf9d8..4c63d857c 100644 --- a/lass/2configs/buildbot-standalone.nix +++ b/lass/2configs/buildbot-standalone.nix @@ -64,7 +64,7 @@ in { # prepare nix-shell # the dependencies which are used by the test script - deps = [ "gnumake", "jq", "nix", "rsync", "proot" ] + deps = [ "gnumake", "jq", "nix", "(import <stockholm>).pkgs.populate" ] # TODO: --pure , prepare ENV in nix-shell command: # SSL_CERT_FILE,LOGNAME,NIX_REMOTE nixshell = ["nix-shell", @@ -112,8 +112,7 @@ in { for i in [ "prism", "mors", "echelon" ]: addShell(f,name="populate-{}".format(i),env=env_lass, command=nixshell + \ - ["{}( make system={} eval.config.krebs.build.populate \ - | jq -er .)".format("!" if "failing" in i else "",i)]) + ["{}(make system={} populate debug=true)".format("!" if "failing" in i else "",i)]) addShell(f,name="build-test-minimal",env=env_lass, command=nixshell + \ @@ -146,7 +145,7 @@ in { masterhost = "localhost"; username = "testslave"; password = "lasspass"; - packages = with pkgs;[ git nix gnumake jq rsync ]; + packages = with pkgs; [ gnumake jq nix populate ]; extraEnviron = { NIX_PATH="nixpkgs=/var/src/nixpkgs"; }; diff --git a/lass/2configs/default.nix b/lass/2configs/default.nix index e3065ba84..b8c50f1aa 100644 --- a/lass/2configs/default.nix +++ b/lass/2configs/default.nix @@ -53,16 +53,14 @@ with config.krebs.lib; search-domain = "retiolum"; build = { user = config.krebs.users.lass; - source = mapAttrs (_: mkDefault) ({ - nixos-config = "symlink:stockholm/lass/1systems/${config.krebs.build.host.name}.nix"; - secrets = if getEnv "dummy_secrets" == "true" - then toString <stockholm/lass/2configs/tests/dummy-secrets> - else "/home/lass/secrets/${config.krebs.build.host.name}"; - #secrets-common = "/home/lass/secrets/common"; - stockholm = getEnv "PWD"; - } // optionalAttrs config.krebs.build.host.secure { - #secrets-master = "/home/lass/secrets/master"; - }); + source = let inherit (config.krebs.build) host; in { + nixos-config.symlink = "stockholm/lass/1systems/${host.name}.nix"; + secrets.file = + if getEnv "dummy_secrets" == "true" + then toString <stockholm/lass/2configs/tests/dummy-secrets> + else "/home/lass/secrets/${host.name}"; + stockholm.file = getEnv "PWD"; + }; }; }; diff --git a/makefu/2configs/default.nix b/makefu/2configs/default.nix index 422927b28..58a537a2b 100644 --- a/makefu/2configs/default.nix +++ b/makefu/2configs/default.nix @@ -19,20 +19,22 @@ with config.krebs.lib; dns.providers.siem = "hosts"; search-domain = "retiolum"; - build = { + build = { user = config.krebs.users.makefu; - source = mapAttrs (_: mkDefault) { - nixpkgs = { + source = let inherit (config.krebs.build) host user; in { + nixpkgs.git = { url = https://github.com/nixos/nixpkgs; - rev = "63b9785"; # stable @ 2016-06-01 + ref = "63b9785"; # stable @ 2016-06-01 }; - secrets = if getEnv "dummy_secrets" == "true" - then toString <stockholm/makefu/6tests/data/secrets> - else "/home/makefu/secrets/${config.krebs.build.host.name}"; - stockholm = "/home/makefu/stockholm"; + secrets.file = + if getEnv "dummy_secrets" == "true" + then toString <stockholm/makefu/6tests/data/secrets> + else "/home/makefu/secrets/${host.name}"; + stockholm.file = "/home/makefu/stockholm"; # Defaults for all stockholm users? - nixos-config = "symlink:stockholm/${config.krebs.build.user.name}/1systems/${config.krebs.build.host.name}.nix"; + nixos-config.symlink = + "stockholm/${user.name}/1systems/${host.name}.nix"; }; }; }; diff --git a/shared/2configs/base.nix b/shared/2configs/base.nix index bbb089c2c..a92a0df35 100644 --- a/shared/2configs/base.nix +++ b/shared/2configs/base.nix @@ -7,15 +7,14 @@ with config.krebs.lib; # TODO rename shared user to "krebs" krebs.build.user = mkDefault config.krebs.users.shared; - krebs.build.source = { - nixpkgs = mkDefault { + krebs.build.source = let inherit (config.krebs.build) host user; in { + nixos-config.symlink = "stockholm/${user.name}/1systems/${host.name}.nix"; + nixpkgs.git = { url = https://github.com/NixOS/nixpkgs; - rev = "63b9785"; # stable @ 2016-06-01 + ref = "63b9785"; # stable @ 2016-06-01 }; - secrets = mkDefault "${getEnv "HOME"}/secrets/krebs/${config.krebs.build.host.name}"; - stockholm = mkDefault "${getEnv "HOME"}/stockholm"; - - nixos-config = "symlink:stockholm/${config.krebs.build.user.name}/1systems/${config.krebs.build.host.name}.nix"; + secrets.file = "${getEnv "HOME"}/secrets/krebs/${host.name}"; + stockholm.file = "${getEnv "HOME"}/stockholm"; }; networking.hostName = config.krebs.build.host.name; diff --git a/shared/2configs/shared-buildbot.nix b/shared/2configs/shared-buildbot.nix index 6c40d9966..688f8f9aa 100644 --- a/shared/2configs/shared-buildbot.nix +++ b/shared/2configs/shared-buildbot.nix @@ -75,7 +75,8 @@ # prepare nix-shell # the dependencies which are used by the test script - deps = [ "gnumake", "jq","nix","rsync", + deps = [ "gnumake", "jq", "nix", + "(import <stockholm>).pkgs.populate", "(import <stockholm>).pkgs.test.infest-cac-centos7" ] # TODO: --pure , prepare ENV in nix-shell command: # SSL_CERT_FILE,LOGNAME,NIX_REMOTE @@ -95,8 +96,7 @@ for i in [ "test-centos7", "wolf", "test-failing" ]: addShell(f,name="populate-{}".format(i),env=env, command=nixshell + \ - ["{}( make system={} eval.config.krebs.build.populate \ - | jq -er .)".format("!" if "failing" in i else "",i)]) + ["{}(make system={} populate debug=true)".format("!" if "failing" in i else "",i)]) # XXX we must prepare ./retiolum.rsa_key.priv for secrets to work addShell(f,name="instantiate-test-all-modules",env=env, @@ -179,7 +179,7 @@ masterhost = "localhost"; username = "testslave"; password = "krebspass"; - packages = with pkgs;[ git nix gnumake jq rsync ]; + packages = with pkgs; [ gnumake jq nix populate ]; # all nix commands will need a working nixpkgs installation extraEnviron = { NIX_PATH="nixpkgs=/var/src/nixpkgs:nixos-config=./shared/1systems/wolf.nix"; }; diff --git a/tv/2configs/default.nix b/tv/2configs/default.nix index a9ba1eadd..04009f54d 100644 --- a/tv/2configs/default.nix +++ b/tv/2configs/default.nix @@ -7,18 +7,18 @@ with config.krebs.lib; krebs.build = { user = config.krebs.users.tv; - source = mapAttrs (_: mkDefault) ({ - nixos-config = "symlink:stockholm/tv/1systems/${config.krebs.build.host.name}.nix"; - secrets = "/home/tv/secrets/${config.krebs.build.host.name}"; - secrets-common = "/home/tv/secrets/common"; - stockholm = "/home/tv/stockholm"; - nixpkgs = { + source = let inherit (config.krebs.build) host; in { + nixos-config.symlink = "stockholm/tv/1systems/${host.name}.nix"; + secrets.file = "/home/tv/secrets/${host.name}"; + secrets-common.file = "/home/tv/secrets/common"; + stockholm.file = "/home/tv/stockholm"; + nixpkgs.git = { url = https://github.com/NixOS/nixpkgs; - rev = "8bf31d7d27cae435d7c1e9e0ccb0a320b424066f"; + ref = "8bf31d7d27cae435d7c1e9e0ccb0a320b424066f"; }; - } // optionalAttrs config.krebs.build.host.secure { - secrets-master = "/home/tv/secrets/master"; - }); + } // optionalAttrs host.secure { + secrets-master.file = "/home/tv/secrets/master"; + }; }; networking.hostName = config.krebs.build.host.name; |