From 0457cd1bb9072dbed13ad74d41ffccd04d8dac20 Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 15 Feb 2016 14:01:20 +0100 Subject: k 3 repo-sync: init module, add git dependency --- krebs/3modules/default.nix | 1 + krebs/3modules/repo-sync.nix | 110 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 krebs/3modules/repo-sync.nix (limited to 'krebs/3modules') diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix index 3d51076cf..060b4445d 100644 --- a/krebs/3modules/default.nix +++ b/krebs/3modules/default.nix @@ -29,6 +29,7 @@ let ./retiolum.nix ./tinc_graphs.nix ./urlwatch.nix + ./repo-sync.nix ]; options.krebs = api; config = mkIf cfg.enable imp; diff --git a/krebs/3modules/repo-sync.nix b/krebs/3modules/repo-sync.nix new file mode 100644 index 000000000..c92d458dd --- /dev/null +++ b/krebs/3modules/repo-sync.nix @@ -0,0 +1,110 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.krebs.repo-sync; + + out = { + options.krebs.repo-sync = api; + config = mkIf cfg.enable imp; + }; + + api = { + enable = mkEnableOption "repo-sync"; + config = mkOption { + type = with types;attrsOf (attrsOf (attrsOf str)); + example = literalExample '' + # see `repo-sync --help` + # `ref` provides sane defaults and can be omitted + + # attrset will be converted to json and be used as config + { + makefu = { + origin = { + url = http://github.com/makefu/repo ; + ref = "heads/dev" ; + }; + mirror = { + url = "git@internal:mirror" ; + ref = "heads/github-mirror-dev" ; + }; + }; + lass = { + origin = { + url = http://github.com/lass/repo ; + }; + mirror = { + url = "git@internal:mirror" ; + }; + }; + "@latest" = { + mirror = { + url = "git@internal:mirror"; + ref = "heads/master"; + }; + }; + }; + ''; + }; + timerConfig = mkOption { + type = types.attrsOf types.str; + default = { + OnCalendar = "*:00,15,30,45"; + }; + }; + stateDir = mkOption { + type = types.str; + default = "/var/lib/repo-sync"; + }; + privateKeyFile = mkOption { + type = types.str; + description = '' + used by repo-sync to identify with ssh service + ''; + default = toString ; + }; + }; + repo-sync-config = pkgs.writeText "repo-sync-config.json" + (builtins.toJSON cfg.config); + + imp = { + users.users.repo-sync = { + name = "repo-sync"; + uid = genid "repo-sync"; + description = "repo-sync user"; + home = cfg.stateDir; + createHome = true; + }; + + systemd.timers.repo-sync = { + description = "repo-sync timer"; + wantedBy = [ "timers.target" ]; + + timerConfig = cfg.timerConfig; + }; + systemd.services.repo-sync = { + description = "repo-sync"; + after = [ "network.target" ]; + + path = with pkgs; [ ]; + + environment = { + GIT_SSH_COMMAND = "${pkgs.openssh}/bin/ssh -i ${cfg.stateDir}/ssh.priv"; + }; + + serviceConfig = { + Type = "simple"; + PermissionsStartOnly = true; + ExecStartPre = pkgs.writeScript "prepare-repo-sync-user" '' + #! /bin/sh + cp -v ${lib.shell.escape cfg.privateKeyFile} ${cfg.stateDir}/ssh.priv + chown repo-sync ${cfg.stateDir}/ssh.priv + ''; + ExecStart = "${pkgs.repo-sync}/bin/repo-sync ${repo-sync-config}"; + WorkingDirectory = cfg.stateDir; + User = "repo-sync"; + }; + }; + }; +in out -- cgit v1.2.3 From a94a4c42065fb2fd489a03fd7b0db60ebabb8ebf Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 15 Feb 2016 17:43:30 +0100 Subject: s 1 wolf: use config.krebs.lib --- krebs/3modules/buildbot/master.nix | 4 ++-- krebs/3modules/buildbot/slave.nix | 6 +++--- krebs/3modules/repo-sync.nix | 5 ++--- 3 files changed, 7 insertions(+), 8 deletions(-) (limited to 'krebs/3modules') diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index 825cb3413..080a1f33d 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -338,8 +338,8 @@ let SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; }; serviceConfig = let - workdir="${lib.shell.escape cfg.workDir}"; - secretsdir="${lib.shell.escape (toString )}"; + workdir="${shell.escape cfg.workDir}"; + secretsdir="${shell.escape (toString )}"; in { PermissionsStartOnly = true; Type = "forking"; diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix index 7705ac31c..0375e8023 100644 --- a/krebs/3modules/buildbot/slave.nix +++ b/krebs/3modules/buildbot/slave.nix @@ -149,9 +149,9 @@ let } // cfg.extraEnviron; serviceConfig = let - workdir = "${lib.shell.escape cfg.workDir}"; - contact = "${lib.shell.escape cfg.contact}"; - description = "${lib.shell.escape cfg.description}"; + workdir = "${shell.escape cfg.workDir}"; + contact = "${shell.escape cfg.contact}"; + description = "${shell.escape cfg.description}"; buildbot = pkgs.buildbot-slave; # TODO:make this in { diff --git a/krebs/3modules/repo-sync.nix b/krebs/3modules/repo-sync.nix index c92d458dd..7a7c80a75 100644 --- a/krebs/3modules/repo-sync.nix +++ b/krebs/3modules/repo-sync.nix @@ -1,7 +1,6 @@ { config, lib, pkgs, ... }: with lib; - let cfg = config.krebs.repo-sync; @@ -71,7 +70,7 @@ let imp = { users.users.repo-sync = { name = "repo-sync"; - uid = genid "repo-sync"; + uid = config.krebs.lib.genid "repo-sync"; description = "repo-sync user"; home = cfg.stateDir; createHome = true; @@ -98,7 +97,7 @@ let PermissionsStartOnly = true; ExecStartPre = pkgs.writeScript "prepare-repo-sync-user" '' #! /bin/sh - cp -v ${lib.shell.escape cfg.privateKeyFile} ${cfg.stateDir}/ssh.priv + cp -v ${config.krebs.lib.shell.escape cfg.privateKeyFile} ${cfg.stateDir}/ssh.priv chown repo-sync ${cfg.stateDir}/ssh.priv ''; ExecStart = "${pkgs.repo-sync}/bin/repo-sync ${repo-sync-config}"; -- cgit v1.2.3 From ea910d7d99ec0d36f7f0cc07566dc82ea16f02ca Mon Sep 17 00:00:00 2001 From: tv Date: Mon, 15 Feb 2016 18:46:19 +0100 Subject: buildbot: s/lib\.shell/shell/g --- krebs/3modules/buildbot/master.nix | 4 ++-- krebs/3modules/buildbot/slave.nix | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'krebs/3modules') diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index 825cb3413..2a1dbe31a 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -338,8 +338,8 @@ let SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; }; serviceConfig = let - workdir="${lib.shell.escape cfg.workDir}"; - secretsdir="${lib.shell.escape (toString )}"; + workdir = shell.escape cfg.workDir; + secretsdir = shell.escape (toString ); in { PermissionsStartOnly = true; Type = "forking"; diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix index 7705ac31c..248b46132 100644 --- a/krebs/3modules/buildbot/slave.nix +++ b/krebs/3modules/buildbot/slave.nix @@ -149,9 +149,9 @@ let } // cfg.extraEnviron; serviceConfig = let - workdir = "${lib.shell.escape cfg.workDir}"; - contact = "${lib.shell.escape cfg.contact}"; - description = "${lib.shell.escape cfg.description}"; + workdir = shell.escape cfg.workDir; + contact = shell.escape cfg.contact; + description = shell.escape cfg.description; buildbot = pkgs.buildbot-slave; # TODO:make this in { -- cgit v1.2.3 From 064d0111a0f56b813dba23ca0b19438f0c2e0530 Mon Sep 17 00:00:00 2001 From: tv Date: Tue, 16 Feb 2016 05:09:09 +0100 Subject: krebs.build: refactor a bit --- krebs/3modules/build.nix | 117 +++++++++++++++++++---------------------------- 1 file changed, 48 insertions(+), 69 deletions(-) (limited to 'krebs/3modules') diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix index a1f446188..c700fbc56 100644 --- a/krebs/3modules/build.nix +++ b/krebs/3modules/build.nix @@ -20,35 +20,19 @@ let type = types.user; }; - options.krebs.build.source = let - raw = types.either types.str types.path; - url = types.submodule { + options.krebs.build.source = mkOption { + type = with types; attrsOf (either str (submodule { options = { - url = mkOption { - type = types.str; - }; - rev = mkOption { - type = types.str; - }; - dev = mkOption { - type = types.str; - }; + url = str; + rev = str; }; - }; - in mkOption { - type = types.attrsOf (types.either types.str url); - apply = let f = mapAttrs (_: value: { - string = value; - path = toString value; - set = f value; - }.${typeOf value}); in f; + })); default = {}; }; options.krebs.build.populate = mkOption { type = types.str; default = let - source = config.krebs.build.source; target-user = maybeEnv "target_user" "root"; target-host = maybeEnv "target_host" config.krebs.build.host.name; target-port = maybeEnv "target_port" "22"; @@ -75,24 +59,21 @@ let tmpdir=$(mktemp -dt stockholm.XXXXXXXX) chmod 0755 "$tmpdir" - ${concatStringsSep "\n" - (mapAttrsToList - (name: spec: let dst = removePrefix "symlink:" (get-url spec); in - "verbose ln -s ${shell.escape dst} $tmpdir/${shell.escape name}") - symlink-specs)} + ${concatStringsSep "\n" (mapAttrsToList (name: symlink: '' + verbose ln -s ${shell.escape symlink.target} \ + "$tmpdir"/${shell.escape name} + '') source-by-method.symlink)} verbose proot \ - -b $tmpdir:${shell.escape target-path} \ - ${concatStringsSep " \\\n " - (mapAttrsToList - (name: spec: - "-b ${shell.escape "${get-url spec}:${target-path}/${name}"}") - file-specs)} \ + -b "$tmpdir":${shell.escape target-path} \ + ${concatStringsSep " \\\n " (mapAttrsToList (name: file: + "-b ${shell.escape "${file.path}:${target-path}/${name}"}" + ) source-by-method.file)} \ rsync \ -f ${shell.escape "P /*"} \ - ${concatMapStringsSep " \\\n " - (name: "-f ${shell.escape "R /${name}"}") - (attrNames file-specs)} \ + ${concatMapStringsSep " \\\n " (name: + "-f ${shell.escape "R /${name}"}" + ) (attrNames source-by-method.file)} \ --delete \ -vFrlptD \ -e ${shell.escape "ssh -p ${target-port}"} \ @@ -100,30 +81,6 @@ let ${shell.escape "${target-user}@${target-host}:${target-path}"} ''; - get-schema = uri: - if substring 0 1 uri == "/" - then "file" - else head (splitString ":" uri); - - has-schema = schema: uri: get-schema uri == schema; - - get-url = spec: { - string = spec; - path = toString spec; - set = get-url spec.url; - }.${typeOf spec}; - - git-specs = - filterAttrs (_: spec: has-schema "https" (get-url spec)) source // - filterAttrs (_: spec: has-schema "http" (get-url spec)) source // - filterAttrs (_: spec: has-schema "git" (get-url spec)) source; - - file-specs = - filterAttrs (_: spec: has-schema "file" (get-url spec)) source; - - symlink-specs = - filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source; - git-script = '' #! /bin/sh set -efu @@ -162,20 +119,42 @@ let git clean -dxf )} - ${concatStringsSep "\n" - (mapAttrsToList - (name: spec: toString (map shell.escape [ - "verbose" - "fetch_git" - "${target-path}/${name}" - spec.url - spec.rev - ])) - git-specs)} + ${concatStringsSep "\n" (mapAttrsToList (name: git: '' + verbose fetch_git ${concatMapStringsSep " " shell.escape [ + "${target-path}/${name}" + git.url + git.rev + ]} + '') source-by-method.git)} ''; in out; }; }; + source-by-method = let + known-methods = ["git" "file" "symlink"]; + in genAttrs known-methods (const {}) // recursiveUpdate source-by-scheme { + git = source-by-scheme.http or {} // + source-by-scheme.https or {}; + }; + + source-by-scheme = foldl' (out: { k, v }: recursiveUpdate out { + ${v.scheme}.${k} = v; + }) {} (mapAttrsToList (k: v: { inherit k v; }) normalized-source); + + normalized-source = mapAttrs (name: let f = x: getAttr (typeOf x) { + path = f (toString x); + string = f { + url = if substring 0 1 x == "/" then "file://${x}" else x; + }; + set = let scheme = head (splitString ":" x.url); in recursiveUpdate x { + inherit scheme; + } // { + symlink.target = removePrefix "symlink:" x.url; + file.path = # TODO file://host/... + assert hasPrefix "file:///" x.url; + removePrefix "file://" x.url; + }.${scheme} or {}; + }; in f) config.krebs.build.source; in out -- cgit v1.2.3