From ecb0d3dd6e97ed01f2ec25710a15ab4497402352 Mon Sep 17 00:00:00 2001 From: makefu Date: Fri, 12 Feb 2016 02:11:56 +0100 Subject: k 5 repo-sync: 0.1.1 -> 0.2.0 --- krebs/5pkgs/repo-sync/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/krebs/5pkgs/repo-sync/default.nix b/krebs/5pkgs/repo-sync/default.nix index 90f838de9..8d4f009eb 100644 --- a/krebs/5pkgs/repo-sync/default.nix +++ b/krebs/5pkgs/repo-sync/default.nix @@ -1,7 +1,7 @@ { lib, pkgs, python3Packages, fetchurl, ... }: with python3Packages; buildPythonPackage rec { name = "repo-sync-${version}"; - version = "0.1.1"; + version = "0.2.0"; disabled = isPy26 || isPy27; propagatedBuildInputs = [ docopt @@ -9,7 +9,7 @@ with python3Packages; buildPythonPackage rec { ]; src = fetchurl { url = "https://pypi.python.org/packages/source/r/repo-sync/repo-sync-${version}.tar.gz"; - sha256 = "01r30l2bbsld90ps13ip0zi2a41b53dv4q6fxrzvkfrprr64c0vv"; + sha256 = "161ssq4138c327p5d68vy91psldal7vp61vk3xdqkhpzd2nz5ag5"; }; meta = { homepage = http://github.com/makefu/repo-sync; -- cgit v1.2.3 From 08b03ef2e7dc6eff237c213ee341d0da6b9a0d96 Mon Sep 17 00:00:00 2001 From: makefu Date: Fri, 12 Feb 2016 02:41:38 +0100 Subject: k 5 repo-sync: 0.2.0 -> 0.2.5 --- krebs/5pkgs/repo-sync/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/krebs/5pkgs/repo-sync/default.nix b/krebs/5pkgs/repo-sync/default.nix index 8d4f009eb..28fc3970d 100644 --- a/krebs/5pkgs/repo-sync/default.nix +++ b/krebs/5pkgs/repo-sync/default.nix @@ -1,7 +1,7 @@ { lib, pkgs, python3Packages, fetchurl, ... }: with python3Packages; buildPythonPackage rec { name = "repo-sync-${version}"; - version = "0.2.0"; + version = "0.2.5"; disabled = isPy26 || isPy27; propagatedBuildInputs = [ docopt @@ -9,7 +9,7 @@ with python3Packages; buildPythonPackage rec { ]; src = fetchurl { url = "https://pypi.python.org/packages/source/r/repo-sync/repo-sync-${version}.tar.gz"; - sha256 = "161ssq4138c327p5d68vy91psldal7vp61vk3xdqkhpzd2nz5ag5"; + sha256 = "1a59bj0vc5ajq8indkvkdk022yzvvv5mjb57hk3xf1j3wpr85p84"; }; meta = { homepage = http://github.com/makefu/repo-sync; -- cgit v1.2.3 From 603752e1e3fe96bdaa9f8e5ffceae6a99a145139 Mon Sep 17 00:00:00 2001 From: makefu Date: Fri, 12 Feb 2016 22:15:18 +0100 Subject: s 2 buildbot: fix regex --- shared/2configs/shared-buildbot.nix | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/shared/2configs/shared-buildbot.nix b/shared/2configs/shared-buildbot.nix index af877f5d8..604cbc5b2 100644 --- a/shared/2configs/shared-buildbot.nix +++ b/shared/2configs/shared-buildbot.nix @@ -26,13 +26,12 @@ builderNames=["full-tests"])) ''; fast-tests-scheduler = '' - # test the master real quick + # test everything BUT the master real quick sched.append(schedulers.SingleBranchScheduler( ## all branches - change_filter=util.ChangeFilter(branch_re=".*"), - # change_filter=util.ChangeFilter(branch="master"), - treeStableTimer=10, #only test the latest push - name="fast-master-test", + change_filter=util.ChangeFilter(branch_re="(?!^master$)"), + # treeStableTimer=10, + name="fast-test-all-branches", builderNames=["fast-tests"])) ''; test-cac-infest-master = '' -- cgit v1.2.3 From e62a0475cd45e30f10d4bce8837b8a776eeb4754 Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 15 Feb 2016 14:00:21 +0100 Subject: s 2 cgit-mirror: add correct pubkey, add user to krebs.users --- shared/2configs/cgit-mirror.nix | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/shared/2configs/cgit-mirror.nix b/shared/2configs/cgit-mirror.nix index 4ff1902f9..0794ee411 100644 --- a/shared/2configs/cgit-mirror.nix +++ b/shared/2configs/cgit-mirror.nix @@ -3,7 +3,7 @@ with lib; let rules = with git; singleton { - user = [ git-sync ]; + user = [ wolf-repo-sync ]; repo = [ stockholm-mirror ]; perm = push ''refs/*'' [ non-fast-forward create delete merge ]; }; @@ -22,14 +22,15 @@ let }; }; - git-sync = { - name = "git-sync"; + wolf-repo-sync = { + name = "wolf-repo-sync"; mail = "spam@krebsco.de"; # TODO put git-sync pubkey somewhere more appropriate - pubkey = ''ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCzUuzyoAhMgJmsiaTVWNSXqcrZNTpKpv0nfFBOMcNXUWEbvfAq5eNpg5cX+P8eoYl6UQgfftbYi06flKK3yJdntxoZKLwJGgJt9NZr8yZTsiIfMG8XosvGNQtGPkBtpLusgmPpu7t2RQ9QrqumBvoUDGYEauKTslLwupp1QeyWKUGEhihn4CuqQKiPrz+9vbNd75XOfVZMggk3j4F7HScatmA+p1EQXWyq5Jj78jQN5ZIRnHjMQcIZ4DOz1U96atwSKMviI1xEZIODYfgoGjjiWYeEtKaLVPtSqtLRGI7l+RNouMfwHLdTWOJSlIdFncfPXC6R19hTll3UHeHLtqLP git-sync''; + pubkey = ''ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwuAZB3wtAvBJFYh+gWdyGaZU4mtqM2dFXmh2rORlbXeh02msu1uv07ck1VKkQ4LgvCBcBsAOeVa1NTz99eLqutwgcqMCytvRNUCibcoEWwHObsK53KhDJj+zotwlFhnPPeK9+EpOP4ngh/tprJikttos5BwBwe2K+lfiid3fmVPZcTTYa77nCwijimMvWEx6CEjq1wiXMUc4+qcEn8Swbwomz/EEQdNE2hgoC3iMW9RqduTFdIJWnjVi0KaxenX9CvQRGbVK5SSu2gwzN59D/okQOCP6+p1gL5r3QRHSLSSRiEHctVQTkpKOifrtLZGSr5zArEmLd/cOVyssHQPCX repo-sync@wolf''; }; in { + krebs.users.wolf-repo-sync = wolf-repo-sync; krebs.git = { enable = true; root-title = "Shared Repos"; -- cgit v1.2.3 From 0457cd1bb9072dbed13ad74d41ffccd04d8dac20 Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 15 Feb 2016 14:01:20 +0100 Subject: k 3 repo-sync: init module, add git dependency --- krebs/3modules/default.nix | 1 + krebs/3modules/repo-sync.nix | 110 ++++++++++++++++++++++++++++++++++++++ krebs/5pkgs/repo-sync/default.nix | 2 + 3 files changed, 113 insertions(+) create mode 100644 krebs/3modules/repo-sync.nix diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix index 3d51076cf..060b4445d 100644 --- a/krebs/3modules/default.nix +++ b/krebs/3modules/default.nix @@ -29,6 +29,7 @@ let ./retiolum.nix ./tinc_graphs.nix ./urlwatch.nix + ./repo-sync.nix ]; options.krebs = api; config = mkIf cfg.enable imp; diff --git a/krebs/3modules/repo-sync.nix b/krebs/3modules/repo-sync.nix new file mode 100644 index 000000000..c92d458dd --- /dev/null +++ b/krebs/3modules/repo-sync.nix @@ -0,0 +1,110 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.krebs.repo-sync; + + out = { + options.krebs.repo-sync = api; + config = mkIf cfg.enable imp; + }; + + api = { + enable = mkEnableOption "repo-sync"; + config = mkOption { + type = with types;attrsOf (attrsOf (attrsOf str)); + example = literalExample '' + # see `repo-sync --help` + # `ref` provides sane defaults and can be omitted + + # attrset will be converted to json and be used as config + { + makefu = { + origin = { + url = http://github.com/makefu/repo ; + ref = "heads/dev" ; + }; + mirror = { + url = "git@internal:mirror" ; + ref = "heads/github-mirror-dev" ; + }; + }; + lass = { + origin = { + url = http://github.com/lass/repo ; + }; + mirror = { + url = "git@internal:mirror" ; + }; + }; + "@latest" = { + mirror = { + url = "git@internal:mirror"; + ref = "heads/master"; + }; + }; + }; + ''; + }; + timerConfig = mkOption { + type = types.attrsOf types.str; + default = { + OnCalendar = "*:00,15,30,45"; + }; + }; + stateDir = mkOption { + type = types.str; + default = "/var/lib/repo-sync"; + }; + privateKeyFile = mkOption { + type = types.str; + description = '' + used by repo-sync to identify with ssh service + ''; + default = toString ; + }; + }; + repo-sync-config = pkgs.writeText "repo-sync-config.json" + (builtins.toJSON cfg.config); + + imp = { + users.users.repo-sync = { + name = "repo-sync"; + uid = genid "repo-sync"; + description = "repo-sync user"; + home = cfg.stateDir; + createHome = true; + }; + + systemd.timers.repo-sync = { + description = "repo-sync timer"; + wantedBy = [ "timers.target" ]; + + timerConfig = cfg.timerConfig; + }; + systemd.services.repo-sync = { + description = "repo-sync"; + after = [ "network.target" ]; + + path = with pkgs; [ ]; + + environment = { + GIT_SSH_COMMAND = "${pkgs.openssh}/bin/ssh -i ${cfg.stateDir}/ssh.priv"; + }; + + serviceConfig = { + Type = "simple"; + PermissionsStartOnly = true; + ExecStartPre = pkgs.writeScript "prepare-repo-sync-user" '' + #! /bin/sh + cp -v ${lib.shell.escape cfg.privateKeyFile} ${cfg.stateDir}/ssh.priv + chown repo-sync ${cfg.stateDir}/ssh.priv + ''; + ExecStart = "${pkgs.repo-sync}/bin/repo-sync ${repo-sync-config}"; + WorkingDirectory = cfg.stateDir; + User = "repo-sync"; + }; + }; + }; +in out diff --git a/krebs/5pkgs/repo-sync/default.nix b/krebs/5pkgs/repo-sync/default.nix index 28fc3970d..789c03f36 100644 --- a/krebs/5pkgs/repo-sync/default.nix +++ b/krebs/5pkgs/repo-sync/default.nix @@ -1,4 +1,5 @@ { lib, pkgs, python3Packages, fetchurl, ... }: + with python3Packages; buildPythonPackage rec { name = "repo-sync-${version}"; version = "0.2.5"; @@ -6,6 +7,7 @@ with python3Packages; buildPythonPackage rec { propagatedBuildInputs = [ docopt GitPython + pkgs.git ]; src = fetchurl { url = "https://pypi.python.org/packages/source/r/repo-sync/repo-sync-${version}.tar.gz"; -- cgit v1.2.3 From ac31ea80288e2f9ae9eda10d28a912e23bc6647e Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 15 Feb 2016 14:02:44 +0100 Subject: s 2 buildbot: use the correct NIX_PATH" --- shared/2configs/shared-buildbot.nix | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/shared/2configs/shared-buildbot.nix b/shared/2configs/shared-buildbot.nix index 604cbc5b2..9327d2f95 100644 --- a/shared/2configs/shared-buildbot.nix +++ b/shared/2configs/shared-buildbot.nix @@ -23,13 +23,13 @@ force-scheduler = '' sched.append(schedulers.ForceScheduler( name="force", - builderNames=["full-tests"])) + builderNames=["full-tests","fast-tests"])) ''; fast-tests-scheduler = '' - # test everything BUT the master real quick + # test everything real quick sched.append(schedulers.SingleBranchScheduler( ## all branches - change_filter=util.ChangeFilter(branch_re="(?!^master$)"), + change_filter=util.ChangeFilter(branch_re=".*"), # treeStableTimer=10, name="fast-test-all-branches", builderNames=["fast-tests"])) @@ -132,7 +132,7 @@ }; irc = { enable = true; - nick = "shared-buildbot"; + nick = "wolfbot"; server = "cd.retiolum"; channels = [ "retiolum" ]; allowForce = true; @@ -146,6 +146,7 @@ password = "krebspass"; packages = with pkgs;[ git nix ]; # all nix commands will need a working nixpkgs installation - extraEnviron = { NIX_PATH="/var/src"; }; + extraEnviron = { + NIX_PATH="nixpkgs=/var/src/upstream-nixpkgs:nixos-config=./shared/1systems/wolf.nix"; }; }; } -- cgit v1.2.3 From 9f2603eb7b1fb4f9161ee896fa5b9081afc3d5d0 Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 15 Feb 2016 14:03:27 +0100 Subject: ma 2: remove krebs.target --- makefu/2configs/default.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/makefu/2configs/default.nix b/makefu/2configs/default.nix index 5acc7f0a5..f7f70de3b 100644 --- a/makefu/2configs/default.nix +++ b/makefu/2configs/default.nix @@ -18,7 +18,6 @@ with lib; enable = true; search-domain = "retiolum"; build = { - target = mkDefault "root@${config.krebs.build.host.name}"; user = config.krebs.users.makefu; source = mapAttrs (_: mkDefault) { upstream-nixpkgs = { -- cgit v1.2.3 From 859144f1d0ca1fd2065f9dfa74cf14cd5af0cc1d Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 15 Feb 2016 14:03:47 +0100 Subject: s 2 repo-sync: init --- shared/1systems/wolf.nix | 1 + shared/2configs/repo-sync.nix | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 shared/2configs/repo-sync.nix diff --git a/shared/1systems/wolf.nix b/shared/1systems/wolf.nix index 9a514428a..588ec1b55 100644 --- a/shared/1systems/wolf.nix +++ b/shared/1systems/wolf.nix @@ -13,6 +13,7 @@ in ../2configs/shack-drivedroid.nix ../2configs/shared-buildbot.nix ../2configs/cgit-mirror.nix + ../2configs/repo-sync.nix # ../2configs/graphite.nix ]; # use your own binary cache, fallback use cache.nixos.org (which is used by diff --git a/shared/2configs/repo-sync.nix b/shared/2configs/repo-sync.nix new file mode 100644 index 000000000..b23cb1675 --- /dev/null +++ b/shared/2configs/repo-sync.nix @@ -0,0 +1,28 @@ +{ config, lib, pkgs, ... }: + +with lib; +{ + krebs.repo-sync = let + # TODO addMirrorURL function + mirror = "git@wolf:stockholm-mirror"; + in { + enable = true; + config = { + makefu = { + origin.url = http://cgit.gum/stockholm ; + mirror.url = mirror; + }; + tv = { + origin.url = http://cgit.cd/stockholm ; + mirror.url = mirror; + }; + lassulus = { + origin.url = http://cgit.cloudkrebs/stockholm ; + mirror.url = mirror; + }; + "@latest" = { + mirror.url = mirror; + }; + }; + }; +} -- cgit v1.2.3 From 9a4071b66ff45e99a30e9a314eb43c6efc7e921f Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 15 Feb 2016 14:25:30 +0100 Subject: s 2 shared-buildbot: add TODO --- shared/2configs/shared-buildbot.nix | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/shared/2configs/shared-buildbot.nix b/shared/2configs/shared-buildbot.nix index 9327d2f95..19724ac01 100644 --- a/shared/2configs/shared-buildbot.nix +++ b/shared/2configs/shared-buildbot.nix @@ -1,18 +1,22 @@ { lib, config, pkgs, ... }: -# The buildbot config is seilf-contained and provides a way to test "shared" -# configuration (infrastructure to be used by every krebsminister). +# The buildbot config is self-contained and currently provides a way +# to test "shared" configuration (infrastructure to be used by every krebsminister). # You can add your own test, test steps as required. Deploy the config on a # shared host like wolf and everything should be fine. + +# TODO for all users schedule a build for fast tests { networking.firewall.allowedTCPPorts = [ 8010 9989 ]; - krebs.buildbot.master = { + krebs.buildbot.master = let + stockholm-mirror-url = http://cgit.wolf/stockholm-mirror ; + in { secrets = [ "retiolum-ci.rsa_key.priv" "cac.json" ]; slaves = { testslave = "krebspass"; }; change_source.stockholm = '' - stockholm_repo = 'http://cgit.wolf/stockholm-mirror' + stockholm_repo = '${stockholm-mirror-url}' cs.append(changes.GitPoller( stockholm_repo, workdir='stockholm-poller', branches=True, -- cgit v1.2.3 From a94a4c42065fb2fd489a03fd7b0db60ebabb8ebf Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 15 Feb 2016 17:43:30 +0100 Subject: s 1 wolf: use config.krebs.lib --- krebs/3modules/buildbot/master.nix | 4 ++-- krebs/3modules/buildbot/slave.nix | 6 +++--- krebs/3modules/repo-sync.nix | 5 ++--- shared/2configs/shack-drivedroid.nix | 4 ++-- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index 825cb3413..080a1f33d 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -338,8 +338,8 @@ let SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; }; serviceConfig = let - workdir="${lib.shell.escape cfg.workDir}"; - secretsdir="${lib.shell.escape (toString )}"; + workdir="${shell.escape cfg.workDir}"; + secretsdir="${shell.escape (toString )}"; in { PermissionsStartOnly = true; Type = "forking"; diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix index 7705ac31c..0375e8023 100644 --- a/krebs/3modules/buildbot/slave.nix +++ b/krebs/3modules/buildbot/slave.nix @@ -149,9 +149,9 @@ let } // cfg.extraEnviron; serviceConfig = let - workdir = "${lib.shell.escape cfg.workDir}"; - contact = "${lib.shell.escape cfg.contact}"; - description = "${lib.shell.escape cfg.description}"; + workdir = "${shell.escape cfg.workDir}"; + contact = "${shell.escape cfg.contact}"; + description = "${shell.escape cfg.description}"; buildbot = pkgs.buildbot-slave; # TODO:make this in { diff --git a/krebs/3modules/repo-sync.nix b/krebs/3modules/repo-sync.nix index c92d458dd..7a7c80a75 100644 --- a/krebs/3modules/repo-sync.nix +++ b/krebs/3modules/repo-sync.nix @@ -1,7 +1,6 @@ { config, lib, pkgs, ... }: with lib; - let cfg = config.krebs.repo-sync; @@ -71,7 +70,7 @@ let imp = { users.users.repo-sync = { name = "repo-sync"; - uid = genid "repo-sync"; + uid = config.krebs.lib.genid "repo-sync"; description = "repo-sync user"; home = cfg.stateDir; createHome = true; @@ -98,7 +97,7 @@ let PermissionsStartOnly = true; ExecStartPre = pkgs.writeScript "prepare-repo-sync-user" '' #! /bin/sh - cp -v ${lib.shell.escape cfg.privateKeyFile} ${cfg.stateDir}/ssh.priv + cp -v ${config.krebs.lib.shell.escape cfg.privateKeyFile} ${cfg.stateDir}/ssh.priv chown repo-sync ${cfg.stateDir}/ssh.priv ''; ExecStart = "${pkgs.repo-sync}/bin/repo-sync ${repo-sync-config}"; diff --git a/shared/2configs/shack-drivedroid.nix b/shared/2configs/shack-drivedroid.nix index 08a6b0697..2e9d2c002 100644 --- a/shared/2configs/shack-drivedroid.nix +++ b/shared/2configs/shack-drivedroid.nix @@ -1,7 +1,8 @@ { pkgs, lib, config, ... }: + let repodir = "/var/srv/drivedroid"; - srepodir = lib.shell.escape repodir; + srepodir = config.krebs.lib.shell.escape repodir; in { environment.systemPackages = [ pkgs.drivedroid-gen-repo ]; @@ -40,5 +41,4 @@ in }; }; }; - } -- cgit v1.2.3