From 6ab41de256066d9870a8f2e260781a9a10365a94 Mon Sep 17 00:00:00 2001 From: lassulus Date: Sun, 13 Dec 2015 15:08:35 +0100 Subject: l 5 xmonad: add binding for mute buttons --- lass/5pkgs/xmonad-lass/Main.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lass/5pkgs/xmonad-lass/Main.hs b/lass/5pkgs/xmonad-lass/Main.hs index ce5afe33a..faaa00aab 100644 --- a/lass/5pkgs/xmonad-lass/Main.hs +++ b/lass/5pkgs/xmonad-lass/Main.hs @@ -125,6 +125,8 @@ myKeyMap = --, ("M4-r", spawn "exe=$(yeganesh -x) && eval \"exec $exe\"") , ("", spawn "pactl -- set-sink-volume 0 +4%") , ("", spawn "pactl -- set-sink-volume 0 -4%") + , ("", spawn "pactl -- set-sink-mute 0 toggle") + , ("", spawn "pactl -- set-source-mute 1 toggle") , ("", gridselectWorkspace myWSConfig W.view) , ("M4-a", focusUrgent) -- cgit v1.2.3 From 9c1207a52825da2f7d9c55304f864d68055cedb8 Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 14 Dec 2015 14:06:05 +0100 Subject: l 5 newsbot-js: rev b227296 -> 6ee4884 --- lass/5pkgs/newsbot-js/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lass/5pkgs/newsbot-js/default.nix b/lass/5pkgs/newsbot-js/default.nix index ace2a976f..0d194e6f3 100644 --- a/lass/5pkgs/newsbot-js/default.nix +++ b/lass/5pkgs/newsbot-js/default.nix @@ -26,8 +26,8 @@ in nodePackages.buildNodePackage { src = fetchgit { url = "http://cgit.echelon/newsbot-js/"; - rev = "b22729670236bfa6491207d57c5d7565137625ca"; - sha256 = "8ff00de56d85543399776c82d41d92ccc68000e5dce0f008d926748e188f3c69"; + rev = "6ee488430c6915eeae03f1569084577d39cef51d"; + sha256 = "00xmn7hzcs0mm6hjf5i37d9nna5rcd0gra0ynch7x2id8liazksx"; }; phases = [ -- cgit v1.2.3 From 57feffb3f65876cca3f10ef82e6e82283c02852d Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 21 Dec 2015 13:47:22 +0100 Subject: l 2 base: nixpkgs rev: 363c843 -> 93d8671 --- lass/2configs/base.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lass/2configs/base.nix b/lass/2configs/base.nix index 40f4e12c7..66e12b262 100644 --- a/lass/2configs/base.nix +++ b/lass/2configs/base.nix @@ -50,7 +50,7 @@ with lib; source = { git.nixpkgs = { url = https://github.com/Lassulus/nixpkgs; - rev = "363c8430f1efad8b03d5feae6b3a4f2fe7b29251"; + rev = "93d8671e2c6d1d25f126ed30e5e6f16764330119"; }; dir.secrets = { host = config.krebs.hosts.mors; -- cgit v1.2.3 From 14d1655deb456d6be95463af2ca1524f7a1b7a98 Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 22 Dec 2015 16:30:01 +0100 Subject: s 1 test-centos7: prepare for ci --- shared/1systems/test-centos7.nix | 3 ++- shared/2configs/base.nix | 4 ++-- shared/2configs/temp/dirs.nix | 1 + shared/2configs/temp/networking.nix | 1 + 4 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 shared/2configs/temp/dirs.nix create mode 100644 shared/2configs/temp/networking.nix diff --git a/shared/1systems/test-centos7.nix b/shared/1systems/test-centos7.nix index 077a5d61b..48cecc877 100644 --- a/shared/1systems/test-centos7.nix +++ b/shared/1systems/test-centos7.nix @@ -7,7 +7,8 @@ in { imports = [ ../2configs/base.nix ../2configs/os-templates/CAC-CentOS-7-64bit.nix - ../2configs/os-templates/temp-networking.nix + ../2configs/temp/networking.nix + ../2configs/temp/dirs.nix ]; sound.enable = false; diff --git a/shared/2configs/base.nix b/shared/2configs/base.nix index df41eae1a..fceea67d1 100644 --- a/shared/2configs/base.nix +++ b/shared/2configs/base.nix @@ -20,11 +20,11 @@ with lib; }; dir.secrets = { host = config.krebs.current.host; - path = "${getEnv "HOME"}/secrets/krebs/wolf"; + path = mkDefault "${getEnv "HOME"}/secrets/krebs/wolf"; }; dir.stockholm = { host = config.krebs.current.host; - path = "${getEnv "HOME"}/stockholm"; + path = mkDefault "${getEnv "HOME"}/stockholm"; }; }; diff --git a/shared/2configs/temp/dirs.nix b/shared/2configs/temp/dirs.nix new file mode 100644 index 000000000..958608a54 --- /dev/null +++ b/shared/2configs/temp/dirs.nix @@ -0,0 +1 @@ +_: { } diff --git a/shared/2configs/temp/networking.nix b/shared/2configs/temp/networking.nix new file mode 100644 index 000000000..958608a54 --- /dev/null +++ b/shared/2configs/temp/networking.nix @@ -0,0 +1 @@ +_: { } -- cgit v1.2.3 From e6b1003fe26e340be21a12e6e531259fd698f33f Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 22 Dec 2015 16:30:23 +0100 Subject: k 5 krebs-ci: initial commit --- krebs/5pkgs/krebs-ci/default.nix | 37 +++++++++++++ krebs/5pkgs/krebs-ci/notes | 111 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 krebs/5pkgs/krebs-ci/default.nix create mode 100755 krebs/5pkgs/krebs-ci/notes diff --git a/krebs/5pkgs/krebs-ci/default.nix b/krebs/5pkgs/krebs-ci/default.nix new file mode 100644 index 000000000..f5b302b52 --- /dev/null +++ b/krebs/5pkgs/krebs-ci/default.nix @@ -0,0 +1,37 @@ +{ stdenv, coreutils,makeWrapper, cac, cacpanel, gnumake, gnused, jq, openssh, ... }: + +stdenv.mkDerivation rec { + name = "krebs-ci-0.1.0"; + + src = ./notes; + + phases = [ + "installPhase" + ]; + buildInputs = [ makeWrapper ]; + + path = stdenv.lib.makeSearchPath "bin" [ + coreutils + cac + cacpanel + gnumake + gnused + jq + openssh + ]; + + installPhase = + '' + mkdir -p $out/bin + cp ${src} $out/bin/krebs-ci + chmod +x $out/bin/krebs-ci + wrapProgram $out/bin/krebs-ci \ + --prefix PATH : ${path} + ''; + meta = with stdenv.lib; { + homepage = http://krebsco.de; + description = "Krebs CI Scripts"; + license = licenses.wtfpl; + maintainers = [ maintainers.makefu ]; + }; +} diff --git a/krebs/5pkgs/krebs-ci/notes b/krebs/5pkgs/krebs-ci/notes new file mode 100755 index 000000000..7e34d6a28 --- /dev/null +++ b/krebs/5pkgs/krebs-ci/notes @@ -0,0 +1,111 @@ +#! /bin/sh + +# nix-shell -p gnumake jq openssh cac cacpanel +set -euf + +# 2 secrets are required: +krebs_cred=${krebs_cred-./cac.json} +retiolum_key=${retiolum_key-./retiolum.rsa_key.priv} + +# Sanity +if test ! -r "$krebs_cred";then + echo "\$krebs_cred=$krebs_cred must be readable"; exit 1 +fi +if test ! -r "$retiolum_key";then + echo "\$retiolum_key=$retiolum_key must be readable"; exit 1 +fi + +krebs_secrets=$(mktemp -d) +sec_file=$krebs_secrets/cac_config +krebs_ssh=$krebs_secrets/tempssh +# we need to receive this key from buildmaster to speed up tinc bootstrap +TRAP="rm $sec_file;rm -r $krebs_secrets" +trap "$TRAP" INT TERM EXIT + +cat > $sec_file <&1\ + | jq -r .servername) + +id=servername:$name +trap "cac delete $id;$TRAP" INT TERM EXIT +# TODO: timeout? +always_update=true cac waitstatus $id "Powered On" + +wait_login_cac(){ + # timeout + for t in `seq 60`;do + # now we have a working cac server + if cac ssh $1 cat /etc/redhat-release | \ + grep CentOS ;then + return 0 + fi + sleep 10 + done + return 1 +} +# die on timeout +wait_login_cac $id + +mkdir -p shared/2configs/temp +cac generatenetworking $id > \ + shared/2configs/temp/networking.nix +# new temporary ssh key we will use to log in after infest +ssh-keygen -f $krebs_ssh -N "" +cp $retiolum_key $krebs_secrets/retiolum.rsa_key.priv +# we override the directories for secrets and stockholm +# additionally we set the ssh key we generated +ip=$(cac getserver $id | jq -r .ip) + +cat > shared/2configs/temp/dirs.nix < $krebs_secrets/infest +sh -x $krebs_secrets/infest + +# TODO: generate secrets directory $krebs_secrets for nix import +cac powerop $id reset + +wait_login(){ + # timeout + for t in `seq 20`;do + # now we have a working cac server + if ssh -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -i $krebs_ssh \ + -o ConnectTimeout=10 \ + -o BatchMode=yes \ + root@$1 nixos-version ;then + return 0 + fi + sleep 10 + done + return 1 +} +wait_login $ip -- cgit v1.2.3 From adbe4a5b4aaed8ea9a7edf20e088f2b74ec3216b Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 22 Dec 2015 16:38:31 +0100 Subject: m 2 default: bump revision --- makefu/2configs/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefu/2configs/default.nix b/makefu/2configs/default.nix index c0d7685e3..a0b49edaf 100644 --- a/makefu/2configs/default.nix +++ b/makefu/2configs/default.nix @@ -24,7 +24,7 @@ with lib; git.nixpkgs = { #url = https://github.com/NixOS/nixpkgs; url = mkDefault https://github.com/makefu/nixpkgs; - rev = mkDefault "78340b042463fd35caa587b0db2e400e5666dbe1"; # nixos-15.09 + cherry-picking + rev = mkDefault "3fd2c24685f604edc925f73ed56600b8c66236b3"; # nixos-15.09 + cherry-picking target-path = "/var/src/nixpkgs"; }; -- cgit v1.2.3 From 9ec4c5df3d132db078e89cb577860b6ec416be04 Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 22 Dec 2015 16:42:28 +0100 Subject: s 2 base: fix user,pubkey,secrets path --- shared/2configs/base.nix | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/shared/2configs/base.nix b/shared/2configs/base.nix index fceea67d1..c36061e38 100644 --- a/shared/2configs/base.nix +++ b/shared/2configs/base.nix @@ -13,6 +13,8 @@ with lib; ]; }; + # TODO rename shared user to "krebs" + krebs.build.user = config.krebs.users.shared; krebs.build.source = { git.nixpkgs = { url = https://github.com/NixOS/nixpkgs; @@ -20,7 +22,7 @@ with lib; }; dir.secrets = { host = config.krebs.current.host; - path = mkDefault "${getEnv "HOME"}/secrets/krebs/wolf"; + path = mkDefault "${getEnv "HOME"}/secrets/krebs/${config.krebs.build.host.name}"; }; dir.stockholm = { host = config.krebs.current.host; @@ -65,7 +67,7 @@ with lib; config.krebs.users.lass.pubkey config.krebs.users.makefu.pubkey # TODO HARDER: - (readFile ../../krebs/Zpubkeys/makefu_omo.ssh.pub) + config.krebs.users.makefu-omo.pubkey config.krebs.users.tv.pubkey ]; -- cgit v1.2.3 From c26ba8d7e674a02995ae613327208f4d9771546b Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 22 Dec 2015 18:53:53 +0100 Subject: m 2 base: build user defaults to shared --- shared/1systems/wolf.nix | 2 -- shared/2configs/base.nix | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/shared/1systems/wolf.nix b/shared/1systems/wolf.nix index 2c51ac8fe..fba4bd9b9 100644 --- a/shared/1systems/wolf.nix +++ b/shared/1systems/wolf.nix @@ -33,8 +33,6 @@ in # uninteresting stuff ##################### krebs.build.host = config.krebs.hosts.wolf; - # TODO rename shared user to "krebs" - krebs.build.user = config.krebs.users.shared; krebs.build.target = "wolf"; boot.kernel.sysctl = { diff --git a/shared/2configs/base.nix b/shared/2configs/base.nix index c36061e38..0ce336558 100644 --- a/shared/2configs/base.nix +++ b/shared/2configs/base.nix @@ -14,7 +14,7 @@ with lib; }; # TODO rename shared user to "krebs" - krebs.build.user = config.krebs.users.shared; + krebs.build.user = mkDefault config.krebs.users.shared; krebs.build.source = { git.nixpkgs = { url = https://github.com/NixOS/nixpkgs; -- cgit v1.2.3 From bf1b6482ce3535ef7e7b3f77879def12ff454c0c Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 22 Dec 2015 19:36:19 +0100 Subject: mv makefu->krebs 3 buildbot --- krebs/3modules/buildbot/master.nix | 263 ++++++++++++++++++++++++++++++++ krebs/3modules/buildbot/slave.nix | 185 ++++++++++++++++++++++ krebs/3modules/default.nix | 2 + makefu/3modules/buildbot/master.nix | 263 -------------------------------- makefu/3modules/buildbot/slave.nix | 185 ---------------------- makefu/3modules/default.nix | 2 - shared/1systems/wolf.nix | 2 +- shared/2configs/buildbot-standalone.nix | 31 ++++ shared/2configs/cac-ci.nix | 11 -- 9 files changed, 482 insertions(+), 462 deletions(-) create mode 100644 krebs/3modules/buildbot/master.nix create mode 100644 krebs/3modules/buildbot/slave.nix delete mode 100644 makefu/3modules/buildbot/master.nix delete mode 100644 makefu/3modules/buildbot/slave.nix create mode 100644 shared/2configs/buildbot-standalone.nix delete mode 100644 shared/2configs/cac-ci.nix diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix new file mode 100644 index 000000000..2f73e44bc --- /dev/null +++ b/krebs/3modules/buildbot/master.nix @@ -0,0 +1,263 @@ +{ config, pkgs, lib, ... }: + +with lib; +let + buildbot = pkgs.buildbot; + buildbot-master-config = pkgs.writeText "buildbot-master.cfg" '' + # -*- python -*- + from buildbot.plugins import * + import re + + c = BuildmasterConfig = {} + + c['slaves'] = [] + # TODO: template potential buildslaves + # TODO: set password? + slavenames= [ 'testslave' ] + for i in slavenames: + c['slaves'].append(buildslave.BuildSlave(i, "krebspass")) + + c['protocols'] = {'pb': {'port': 9989}} + + ####### Build Inputs + stockholm_repo = 'http://cgit.gum/stockholm' + c['change_source'] = [] + c['change_source'].append(changes.GitPoller( + stockholm_repo, + workdir='stockholm-poller', branch='master', + project='stockholm', + pollinterval=120)) + + ####### Build Scheduler + # TODO: configure scheduler + c['schedulers'] = [] + + # test the master real quick + fast = schedulers.SingleBranchScheduler( + change_filter=util.ChangeFilter(branch="master"), + name="fast-master-test", + builderNames=["fast-tests"]) + + force = schedulers.ForceScheduler( + name="force", + builderNames=["full-tests"]) + + # files everyone depends on or are part of the share branch + def shared_files(change): + r =re.compile("^((krebs|share)/.*|Makefile|default.nix)") + for file in change.files: + if r.match(file): + return True + return False + + full = schedulers.SingleBranchScheduler( + change_filter=util.ChangeFilter(branch="master"), + fileIsImportant=shared_files, + name="full-master-test", + builderNames=["full-tests"]) + c['schedulers'] = [ fast, force, full ] + ###### The actual build + # couple of fast steps: + f = util.BuildFactory() + ## fetch repo + grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') + f.addStep(grab_repo) + + # the dependencies which are used by the test script + deps = [ "gnumake", "jq" ] + nixshell = ["nix-shell", "-p" ] + deps + [ "--run" ] + def addShell(f,**kwargs): + f.addStep(steps.ShellCommand(**kwargs)) + + addShell(f,name="centos7-eval",env={"LOGNAME": "shared", + "get" : "krebs.deploy", + "filter" : "json" + }, + command=nixshell + ["make -s eval system=test-centos7"]) + + addShell(f,name="wolf-eval",env={"LOGNAME": "shared", + "get" : "krebs.deploy", + "filter" : "json" + }, + command=nixshell + ["make -s eval system=wolf"]) + + c['builders'] = [] + c['builders'].append( + util.BuilderConfig(name="fast-tests", + slavenames=slavenames, + factory=f)) + + # TODO slow build + c['builders'].append( + util.BuilderConfig(name="full-tests", + slavenames=slavenames, + factory=f)) + + ####### Status of Builds + c['status'] = [] + + from buildbot.status import html + from buildbot.status.web import authz, auth + # TODO: configure if http is wanted + authz_cfg=authz.Authz( + # TODO: configure user/pw + auth=auth.BasicAuth([("krebs","bob")]), + gracefulShutdown = False, + forceBuild = 'auth', + forceAllBuilds = 'auth', + pingBuilder = False, + stopBuild = False, + stopAllBuilds = False, + cancelPendingBuild = False, + ) + # TODO: configure nginx + c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg)) + + from buildbot.status import words + ${optionalString (cfg.irc.enable) '' + irc = words.IRC("${cfg.irc.server}", "krebsbuild", + # TODO: multiple channels + channels=["${cfg.irc.channel}"], + notify_events={ + #'success': 1, + #'failure': 1, + 'exception': 1, + 'successToFailure': 1, + 'failureToSuccess': 1, + }${optionalString cfg.irc.allowForce ",allowForce=True"}) + c['status'].append(irc) + ''} + + ####### PROJECT IDENTITY + c['title'] = "Stockholm" + c['titleURL'] = "http://krebsco.de" + + #c['buildbotURL'] = "http://buildbot.krebsco.de/" + # TODO: configure url + c['buildbotURL'] = "http://vbob:8010/" + + ####### DB URL + c['db'] = { + 'db_url' : "sqlite:///state.sqlite", + } + ${cfg.extraConfig} + ''; + + cfg = config.krebs.buildbot.master; + + api = { + enable = mkEnableOption "Buildbot Master"; + workDir = mkOption { + default = "/var/lib/buildbot/master"; + type = types.str; + description = '' + Path to build bot master directory. + Will be created on startup. + ''; + }; + irc = mkOption { + default = {}; + type = types.submodule ({ config, ... }: { + options = { + enable = mkEnableOption "Buildbot Master IRC Status"; + channel = mkOption { + default = "nix-buildbot-meetup"; + type = types.str; + description = '' + irc channel the bot should connect to + ''; + }; + allowForce = mkOption { + default = false; + type = types.bool; + description = '' + Determines if builds can be forced via IRC + ''; + }; + nick = mkOption { + default = "nix-buildbot"; + type = types.str; + description = '' + nickname for IRC + ''; + }; + server = mkOption { + default = "irc.freenode.net"; + type = types.str; + description = '' + Buildbot Status IRC Server to connect to + ''; + }; + }; + }); + }; + + extraConfig = mkOption { + default = ""; + type = types.lines; + description = '' + extra config appended to the generated master.cfg + ''; + }; + }; + + imp = { + + users.extraUsers.buildbotMaster = { + uid = 672626386; #genid buildbotMaster + description = "Buildbot Master"; + home = cfg.workDir; + createHome = false; + }; + + users.extraGroups.buildbotMaster = { + gid = 672626386; + }; + + systemd.services.buildbotMaster = { + description = "Buildbot Master"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + path = [ pkgs.git ]; + serviceConfig = let + workdir="${lib.shell.escape cfg.workDir}"; + # TODO: check if git is the only dep + in { + PermissionsStartOnly = true; + Type = "forking"; + PIDFile = "${workdir}/twistd.pid"; + # TODO: maybe also prepare buildbot.tac? + ExecStartPre = pkgs.writeScript "buildbot-master-init" '' + #!/bin/sh + set -efux + if [ ! -e ${workdir} ];then + mkdir -p ${workdir} + ${buildbot}/bin/buildbot create-master -r -l 10 -f ${workdir} + fi + # always override the master.cfg + cp ${buildbot-master-config} ${workdir}/master.cfg + # sanity + ${buildbot}/bin/buildbot checkconfig ${workdir} + + # TODO: maybe upgrade? not sure about this + # normally we should write buildbot.tac by our own + # ${buildbot}/bin/buildbot upgrade-master ${workdir} + + chmod 700 -R ${workdir} + chown buildbotMaster:buildbotMaster -R ${workdir} + ''; + ExecStart = "${buildbot}/bin/buildbot start ${workdir}"; + ExecStop = "${buildbot}/bin/buildbot stop ${workdir}"; + ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}"; + PrivateTmp = "true"; + User = "buildbotMaster"; + Restart = "always"; + RestartSec = "10"; + }; + }; + }; +in +{ + options.krebs.buildbot.master = api; + config = mkIf cfg.enable imp; +} diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix new file mode 100644 index 000000000..65291f63e --- /dev/null +++ b/krebs/3modules/buildbot/slave.nix @@ -0,0 +1,185 @@ +{ config, pkgs, lib, ... }: + +with lib; +let + buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" '' + import os + + from buildslave.bot import BuildSlave + from twisted.application import service + + basedir = '${cfg.workDir}' + rotateLength = 10000000 + maxRotatedFiles = 10 + + application = service.Application('buildslave') + + from twisted.python.logfile import LogFile + from twisted.python.log import ILogObserver, FileLogObserver + logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength, + maxRotatedFiles=maxRotatedFiles) + application.setComponent(ILogObserver, FileLogObserver(logfile).emit) + + buildmaster_host = '${cfg.masterhost}' + # TODO: masterport? + port = 9989 + slavename = '${cfg.username}' + passwd = '${cfg.password}' + keepalive = 600 + usepty = 0 + umask = None + maxdelay = 300 + allow_shutdown = None + + ${cfg.extraConfig} + + s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir, + keepalive, usepty, umask=umask, maxdelay=maxdelay, + allow_shutdown=allow_shutdown) + s.setServiceParent(application) + ''; + default-packages = [ pkgs.git pkgs.bash ]; + cfg = config.krebs.buildbot.slave; + + api = { + enable = mkEnableOption "Buildbot Slave"; + + workDir = mkOption { + default = "/var/lib/buildbot/slave"; + type = types.str; + description = '' + Path to build bot slave directory. + Will be created on startup. + ''; + }; + + masterhost = mkOption { + default = "localhost"; + type = types.str; + description = '' + Hostname/IP of the buildbot master + ''; + }; + + username = mkOption { + type = types.str; + description = '' + slavename used to authenticate with master + ''; + }; + + password = mkOption { + type = types.str; + description = '' + slave password used to authenticate with master + ''; + }; + + contact = mkOption { + default = "nix slave "; + type = types.str; + description = '' + contact to be announced by buildslave + ''; + }; + + description = mkOption { + default = "Nix Generated BuildSlave"; + type = types.str; + description = '' + description for hostto be announced by buildslave + ''; + }; + + packages = mkOption { + default = [ pkgs.git ]; + type = with types; listOf package; + description = '' + packages which should be in path for buildslave + ''; + }; + + extraEnviron = mkOption { + default = {}; + example = { + NIX_PATH = "nixpkgs=/path/to/my/nixpkgs"; + }; + type = types.attrsOf types.str; + description = '' + extra environment variables to be provided to the buildslave service + if you need nixpkgs, e.g. for running nix-shell you can set NIX_PATH here. + ''; + }; + + extraConfig = mkOption { + default = ""; + type = types.lines; + example = '' + port = 443 + keepalive = 600 + ''; + description = '' + extra config evaluated before calling BuildSlave init in .tac file + ''; + }; + }; + + imp = { + + users.extraUsers.buildbotSlave = { + uid = 1408105834; #genid buildbotMaster + description = "Buildbot Slave"; + home = cfg.workDir; + createHome = false; + }; + + users.extraGroups.buildbotSlave = { + gid = 1408105834; + }; + + systemd.services."buildbotSlave-${cfg.username}-${cfg.masterhost}" = { + description = "Buildbot Slave for ${cfg.username}@${cfg.masterhost}"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + path = default-packages ++ cfg.packages; + + environment = { + NIX_REMOTE="daemon"; + } // cfg.extraEnviron; + + serviceConfig = let + workdir = "${lib.shell.escape cfg.workDir}"; + contact = "${lib.shell.escape cfg.contact}"; + description = "${lib.shell.escape cfg.description}"; + buildbot = pkgs.buildbot-slave; + # TODO:make this + in { + PermissionsStartOnly = true; + Type = "forking"; + PIDFile = "${workdir}/twistd.pid"; + # TODO: maybe also prepare buildbot.tac? + ExecStartPre = pkgs.writeScript "buildbot-master-init" '' + #!/bin/sh + set -efux + mkdir -p ${workdir}/info + cp ${buildbot-slave-init} ${workdir}/buildbot.tac + echo ${contact} > ${workdir}/info/admin + echo ${description} > ${workdir}/info/host + + chown buildbotSlave:buildbotSlave -R ${workdir} + chmod 700 -R ${workdir} + ''; + ExecStart = "${buildbot}/bin/buildslave start ${workdir}"; + ExecStop = "${buildbot}/bin/buildslave stop ${workdir}"; + PrivateTmp = "true"; + User = "buildbotSlave"; + Restart = "always"; + RestartSec = "10"; + }; + }; + }; +in +{ + options.krebs.buildbot.slave = api; + config = mkIf cfg.enable imp; +} diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix index 740ba67b8..cbc1291fa 100644 --- a/krebs/3modules/default.nix +++ b/krebs/3modules/default.nix @@ -9,6 +9,8 @@ let ./apt-cacher-ng.nix ./bepasty-server.nix ./build.nix + ./buildbot/master.nix + ./buildbot/slave.nix ./current.nix ./exim-retiolum.nix ./exim-smarthost.nix diff --git a/makefu/3modules/buildbot/master.nix b/makefu/3modules/buildbot/master.nix deleted file mode 100644 index 58e2f8175..000000000 --- a/makefu/3modules/buildbot/master.nix +++ /dev/null @@ -1,263 +0,0 @@ -{ config, pkgs, lib, ... }: - -with lib; -let - buildbot = pkgs.buildbot; - buildbot-master-config = pkgs.writeText "buildbot-master.cfg" '' - # -*- python -*- - from buildbot.plugins import * - import re - - c = BuildmasterConfig = {} - - c['slaves'] = [] - # TODO: template potential buildslaves - # TODO: set password? - slavenames= [ 'testslave' ] - for i in slavenames: - c['slaves'].append(buildslave.BuildSlave(i, "krebspass")) - - c['protocols'] = {'pb': {'port': 9989}} - - ####### Build Inputs - stockholm_repo = 'http://cgit.gum/stockholm' - c['change_source'] = [] - c['change_source'].append(changes.GitPoller( - stockholm_repo, - workdir='stockholm-poller', branch='master', - project='stockholm', - pollinterval=120)) - - ####### Build Scheduler - # TODO: configure scheduler - c['schedulers'] = [] - - # test the master real quick - fast = schedulers.SingleBranchScheduler( - change_filter=util.ChangeFilter(branch="master"), - name="fast-master-test", - builderNames=["fast-tests"]) - - force = schedulers.ForceScheduler( - name="force", - builderNames=["full-tests"]) - - # files everyone depends on or are part of the share branch - def shared_files(change): - r =re.compile("^((krebs|share)/.*|Makefile|default.nix)") - for file in change.files: - if r.match(file): - return True - return False - - full = schedulers.SingleBranchScheduler( - change_filter=util.ChangeFilter(branch="master"), - fileIsImportant=shared_files, - name="full-master-test", - builderNames=["full-tests"]) - c['schedulers'] = [ fast, force, full ] - ###### The actual build - # couple of fast steps: - f = util.BuildFactory() - ## fetch repo - grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') - f.addStep(grab_repo) - - # the dependencies which are used by the test script - deps = [ "gnumake", "jq" ] - nixshell = ["nix-shell", "-p" ] + deps + [ "--run" ] - def addShell(f,**kwargs): - f.addStep(steps.ShellCommand(**kwargs)) - - addShell(f,name="centos7-eval",env={"LOGNAME": "shared", - "get" : "krebs.deploy", - "filter" : "json" - }, - command=nixshell + ["make -s eval system=test-centos7"]) - - addShell(f,name="wolf-eval",env={"LOGNAME": "shared", - "get" : "krebs.deploy", - "filter" : "json" - }, - command=nixshell + ["make -s eval system=wolf"]) - - c['builders'] = [] - c['builders'].append( - util.BuilderConfig(name="fast-tests", - slavenames=slavenames, - factory=f)) - - # TODO slow build - c['builders'].append( - util.BuilderConfig(name="full-tests", - slavenames=slavenames, - factory=f)) - - ####### Status of Builds - c['status'] = [] - - from buildbot.status import html - from buildbot.status.web import authz, auth - # TODO: configure if http is wanted - authz_cfg=authz.Authz( - # TODO: configure user/pw - auth=auth.BasicAuth([("krebs","bob")]), - gracefulShutdown = False, - forceBuild = 'auth', - forceAllBuilds = 'auth', - pingBuilder = False, - stopBuild = False, - stopAllBuilds = False, - cancelPendingBuild = False, - ) - # TODO: configure nginx - c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg)) - - from buildbot.status import words - ${optionalString (cfg.irc.enable) '' - irc = words.IRC("${cfg.irc.server}", "krebsbuild", - # TODO: multiple channels - channels=["${cfg.irc.channel}"], - notify_events={ - #'success': 1, - #'failure': 1, - 'exception': 1, - 'successToFailure': 1, - 'failureToSuccess': 1, - }${optionalString cfg.irc.allowForce ",allowForce=True"}) - c['status'].append(irc) - ''} - - ####### PROJECT IDENTITY - c['title'] = "Stockholm" - c['titleURL'] = "http://krebsco.de" - - #c['buildbotURL'] = "http://buildbot.krebsco.de/" - # TODO: configure url - c['buildbotURL'] = "http://vbob:8010/" - - ####### DB URL - c['db'] = { - 'db_url' : "sqlite:///state.sqlite", - } - ${cfg.extraConfig} - ''; - - cfg = config.makefu.buildbot.master; - - api = { - enable = mkEnableOption "Buildbot Master"; - workDir = mkOption { - default = "/var/lib/buildbot/master"; - type = types.str; - description = '' - Path to build bot master directory. - Will be created on startup. - ''; - }; - irc = mkOption { - default = {}; - type = types.submodule ({ config, ... }: { - options = { - enable = mkEnableOption "Buildbot Master IRC Status"; - channel = mkOption { - default = "nix-buildbot-meetup"; - type = types.str; - description = '' - irc channel the bot should connect to - ''; - }; - allowForce = mkOption { - default = false; - type = types.bool; - description = '' - Determines if builds can be forced via IRC - ''; - }; - nick = mkOption { - default = "nix-buildbot"; - type = types.str; - description = '' - nickname for IRC - ''; - }; - server = mkOption { - default = "irc.freenode.net"; - type = types.str; - description = '' - Buildbot Status IRC Server to connect to - ''; - }; - }; - }); - }; - - extraConfig = mkOption { - default = ""; - type = types.lines; - description = '' - extra config appended to the generated master.cfg - ''; - }; - }; - - imp = { - - users.extraUsers.buildbotMaster = { - uid = 672626386; #genid buildbotMaster - description = "Buildbot Master"; - home = cfg.workDir; - createHome = false; - }; - - users.extraGroups.buildbotMaster = { - gid = 672626386; - }; - - systemd.services.buildbotMaster = { - description = "Buildbot Master"; - after = [ "network.target" ]; - wantedBy = [ "multi-user.target" ]; - path = [ pkgs.git ]; - serviceConfig = let - workdir="${lib.shell.escape cfg.workDir}"; - # TODO: check if git is the only dep - in { - PermissionsStartOnly = true; - Type = "forking"; - PIDFile = "${workdir}/twistd.pid"; - # TODO: maybe also prepare buildbot.tac? - ExecStartPre = pkgs.writeScript "buildbot-master-init" '' - #!/bin/sh - set -efux - if [ ! -e ${workdir} ];then - mkdir -p ${workdir} - ${buildbot}/bin/buildbot create-master -r -l 10 -f ${workdir} - fi - # always override the master.cfg - cp ${buildbot-master-config} ${workdir}/master.cfg - # sanity - ${buildbot}/bin/buildbot checkconfig ${workdir} - - # TODO: maybe upgrade? not sure about this - # normally we should write buildbot.tac by our own - # ${buildbot}/bin/buildbot upgrade-master ${workdir} - - chmod 700 -R ${workdir} - chown buildbotMaster:buildbotMaster -R ${workdir} - ''; - ExecStart = "${buildbot}/bin/buildbot start ${workdir}"; - ExecStop = "${buildbot}/bin/buildbot stop ${workdir}"; - ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}"; - PrivateTmp = "true"; - User = "buildbotMaster"; - Restart = "always"; - RestartSec = "10"; - }; - }; - }; -in -{ - options.makefu.buildbot.master = api; - config = mkIf cfg.enable imp; -} diff --git a/makefu/3modules/buildbot/slave.nix b/makefu/3modules/buildbot/slave.nix deleted file mode 100644 index 69d0361bf..000000000 --- a/makefu/3modules/buildbot/slave.nix +++ /dev/null @@ -1,185 +0,0 @@ -{ config, pkgs, lib, ... }: - -with lib; -let - buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" '' - import os - - from buildslave.bot import BuildSlave - from twisted.application import service - - basedir = '${cfg.workDir}' - rotateLength = 10000000 - maxRotatedFiles = 10 - - application = service.Application('buildslave') - - from twisted.python.logfile import LogFile - from twisted.python.log import ILogObserver, FileLogObserver - logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength, - maxRotatedFiles=maxRotatedFiles) - application.setComponent(ILogObserver, FileLogObserver(logfile).emit) - - buildmaster_host = '${cfg.masterhost}' - # TODO: masterport? - port = 9989 - slavename = '${cfg.username}' - passwd = '${cfg.password}' - keepalive = 600 - usepty = 0 - umask = None - maxdelay = 300 - allow_shutdown = None - - ${cfg.extraConfig} - - s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir, - keepalive, usepty, umask=umask, maxdelay=maxdelay, - allow_shutdown=allow_shutdown) - s.setServiceParent(application) - ''; - default-packages = [ pkgs.git pkgs.bash ]; - cfg = config.makefu.buildbot.slave; - - api = { - enable = mkEnableOption "Buildbot Slave"; - - workDir = mkOption { - default = "/var/lib/buildbot/slave"; - type = types.str; - description = '' - Path to build bot slave directory. - Will be created on startup. - ''; - }; - - masterhost = mkOption { - default = "localhost"; - type = types.str; - description = '' - Hostname/IP of the buildbot master - ''; - }; - - username = mkOption { - type = types.str; - description = '' - slavename used to authenticate with master - ''; - }; - - password = mkOption { - type = types.str; - description = '' - slave password used to authenticate with master - ''; - }; - - contact = mkOption { - default = "nix slave "; - type = types.str; - description = '' - contact to be announced by buildslave - ''; - }; - - description = mkOption { - default = "Nix Generated BuildSlave"; - type = types.str; - description = '' - description for hostto be announced by buildslave - ''; - }; - - packages = mkOption { - default = [ pkgs.git ]; - type = with types; listOf package; - description = '' - packages which should be in path for buildslave - ''; - }; - - extraEnviron = mkOption { - default = {}; - example = { - NIX_PATH = "nixpkgs=/path/to/my/nixpkgs"; - }; - type = types.attrsOf types.str; - description = '' - extra environment variables to be provided to the buildslave service - if you need nixpkgs, e.g. for running nix-shell you can set NIX_PATH here. - ''; - }; - - extraConfig = mkOption { - default = ""; - type = types.lines; - example = '' - port = 443 - keepalive = 600 - ''; - description = '' - extra config evaluated before calling BuildSlave init in .tac file - ''; - }; - }; - - imp = { - - users.extraUsers.buildbotSlave = { - uid = 1408105834; #genid buildbotMaster - description = "Buildbot Slave"; - home = cfg.workDir; - createHome = false; - }; - - users.extraGroups.buildbotSlave = { - gid = 1408105834; - }; - - systemd.services."buildbotSlave-${cfg.username}-${cfg.masterhost}" = { - description = "Buildbot Slave for ${cfg.username}@${cfg.masterhost}"; - after = [ "network.target" ]; - wantedBy = [ "multi-user.target" ]; - path = default-packages ++ cfg.packages; - - environment = { - NIX_REMOTE="daemon"; - } // cfg.extraEnviron; - - serviceConfig = let - workdir = "${lib.shell.escape cfg.workDir}"; - contact = "${lib.shell.escape cfg.contact}"; - description = "${lib.shell.escape cfg.description}"; - buildbot = pkgs.buildbot-slave; - # TODO:make this - in { - PermissionsStartOnly = true; - Type = "forking"; - PIDFile = "${workdir}/twistd.pid"; - # TODO: maybe also prepare buildbot.tac? - ExecStartPre = pkgs.writeScript "buildbot-master-init" '' - #!/bin/sh - set -efux - mkdir -p ${workdir}/info - cp ${buildbot-slave-init} ${workdir}/buildbot.tac - echo ${contact} > ${workdir}/info/admin - echo ${description} > ${workdir}/info/host - - chown buildbotSlave:buildbotSlave -R ${workdir} - chmod 700 -R ${workdir} - ''; - ExecStart = "${buildbot}/bin/buildslave start ${workdir}"; - ExecStop = "${buildbot}/bin/buildslave stop ${workdir}"; - PrivateTmp = "true"; - User = "buildbotSlave"; - Restart = "always"; - RestartSec = "10"; - }; - }; - }; -in -{ - options.makefu.buildbot.slave = api; - config = mkIf cfg.enable imp; -} diff --git a/makefu/3modules/default.nix b/makefu/3modules/default.nix index ffbf54cc0..a8a1f69d0 100644 --- a/makefu/3modules/default.nix +++ b/makefu/3modules/default.nix @@ -2,8 +2,6 @@ _: { imports = [ - ./buildbot/master.nix - ./buildbot/slave.nix ]; } diff --git a/shared/1systems/wolf.nix b/shared/1systems/wolf.nix index fba4bd9b9..f05356f0f 100644 --- a/shared/1systems/wolf.nix +++ b/shared/1systems/wolf.nix @@ -11,7 +11,7 @@ in ../2configs/collectd-base.nix ../2configs/shack-nix-cacher.nix ../2configs/shack-drivedroid.nix - ../2configs/cac-ci.nix + ../2configs/buildbot-standalone.nix ../2configs/graphite.nix ]; # use your own binary cache, fallback use cache.nixos.org (which is used by diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix new file mode 100644 index 000000000..adf44cada --- /dev/null +++ b/shared/2configs/buildbot-standalone.nix @@ -0,0 +1,31 @@ +{ lib, config, pkgs, ... }: +let + pkgs-unst = import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz) {}; +in { + nixpkgs.config.packageOverrides = pkgs: { + buildbot = pkgs-unst.buildbot; + buildbot-slave = pkgs-unst.buildbot-slave; + }; + networking.firewall.allowedTCPPorts = [ 8010 ]; + krebs.buildbot.master = { + enable = true; + irc = { + enable = true; + server = "cd.retiolum"; + channel = "retiolum"; + allowForce = true; + }; + extraConfig = '' + c['buildbotURL'] = "http://${config.krebs.build.host.name}:8010/" + ''; + }; + + krebs.buildbot.slave = { + enable = true; + masterhost = "localhost"; + username = "testslave"; + password = "krebspass"; + packages = with pkgs;[ git nix ]; + extraEnviron = { NIX_PATH="nixpkgs=${toString }"; }; + }; +} diff --git a/shared/2configs/cac-ci.nix b/shared/2configs/cac-ci.nix deleted file mode 100644 index 06cce2746..000000000 --- a/shared/2configs/cac-ci.nix +++ /dev/null @@ -1,11 +0,0 @@ -{ config, lib, pkgs, ... }: - -with lib; -{ - environment.systemPackages = with pkgs;[ - get - cac - cacpanel - jq - ]; -} -- cgit v1.2.3 From 6c5921c9fc84211b42a93ab715a25dc7d77a1907 Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 22 Dec 2015 20:31:21 +0100 Subject: Makefile: fail if nix-instantiate fails --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index aefd17147..5b898c54c 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ ifeq ($(filter),json) else filter() { cat; } endif - nix-instantiate \ + result=$$(nix-instantiate \ $${extraArgs-} \ --eval \ -A "$$get" \ @@ -45,8 +45,9 @@ endif --argstr current-host-name "$$HOSTNAME" \ --argstr current-user-name "$$LOGNAME" \ $${system+--argstr system "$$system"} \ - $${target+--argstr target "$$target"} \ - | filter + $${target+--argstr target "$$target"}) + echo "$$result" | filter + else $(error unbound variable: system[s]) endif -- cgit v1.2.3 From 56e8346faa75fc42f65d11ea3569a3e5bdd252ec Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 22 Dec 2015 20:53:11 +0100 Subject: k 5 krebs-ci: remove obsolete trap rm --- krebs/3modules/buildbot/master.nix | 2 +- krebs/5pkgs/krebs-ci/notes | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index 2f73e44bc..e66e0d6b2 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -44,7 +44,7 @@ let # files everyone depends on or are part of the share branch def shared_files(change): - r =re.compile("^((krebs|share)/.*|Makefile|default.nix)") + r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)") for file in change.files: if r.match(file): return True diff --git a/krebs/5pkgs/krebs-ci/notes b/krebs/5pkgs/krebs-ci/notes index 7e34d6a28..f6b193ddb 100755 --- a/krebs/5pkgs/krebs-ci/notes +++ b/krebs/5pkgs/krebs-ci/notes @@ -19,7 +19,7 @@ krebs_secrets=$(mktemp -d) sec_file=$krebs_secrets/cac_config krebs_ssh=$krebs_secrets/tempssh # we need to receive this key from buildmaster to speed up tinc bootstrap -TRAP="rm $sec_file;rm -r $krebs_secrets" +TRAP="rm -r $krebs_secrets" trap "$TRAP" INT TERM EXIT cat > $sec_file < Date: Tue, 22 Dec 2015 22:00:30 +0100 Subject: s 1 test-failing: add for CI --- shared/1systems/test-failing.nix | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 shared/1systems/test-failing.nix diff --git a/shared/1systems/test-failing.nix b/shared/1systems/test-failing.nix new file mode 100644 index 000000000..81a9e48d6 --- /dev/null +++ b/shared/1systems/test-failing.nix @@ -0,0 +1,6 @@ +{ config, pkgs, ... }: + +{ + programs.ssh.startAgent = true; + programs.ssh.startAgent = false; +} -- cgit v1.2.3 From 1a184c98a21ed32447bb4a88f7c865adef5a535f Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 22 Dec 2015 23:37:12 +0100 Subject: k 5 krebs-ci: set cache files manually --- krebs/5pkgs/krebs-ci/notes | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/krebs/5pkgs/krebs-ci/notes b/krebs/5pkgs/krebs-ci/notes index f6b193ddb..f6f3da8db 100755 --- a/krebs/5pkgs/krebs-ci/notes +++ b/krebs/5pkgs/krebs-ci/notes @@ -1,9 +1,10 @@ #! /bin/sh # nix-shell -p gnumake jq openssh cac cacpanel -set -euf +set -eufx # 2 secrets are required: + krebs_cred=${krebs_cred-./cac.json} retiolum_key=${retiolum_key-./retiolum.rsa_key.priv} @@ -18,8 +19,12 @@ fi krebs_secrets=$(mktemp -d) sec_file=$krebs_secrets/cac_config krebs_ssh=$krebs_secrets/tempssh +cac_resources_cache=$krebs_secrets/res_cache.json +cac_servers_cache=$krebs_secrets/servers_cache.json +cac_tasks_cache=$krebs_secrets/tasks_cache.json +cac_templates_cache=$krebs_secrets/templates_cache.json # we need to receive this key from buildmaster to speed up tinc bootstrap -TRAP="rm -r $krebs_secrets" +TRAP="rm -r $krebs_secrets;exit" trap "$TRAP" INT TERM EXIT cat > $sec_file <&1\ id=servername:$name trap "cac delete $id;$TRAP" INT TERM EXIT # TODO: timeout? -always_update=true cac waitstatus $id "Powered On" +# cac_always_update=true cac waitstatus $id "Powered On" wait_login_cac(){ # timeout - for t in `seq 60`;do + for t in `seq 180`;do # now we have a working cac server if cac ssh $1 cat /etc/redhat-release | \ grep CentOS ;then -- cgit v1.2.3 From f59080e76f950a5a8e33d1edd4314ffaa14187fc Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 23 Dec 2015 00:06:27 +0100 Subject: m 3 buildbot: add new slow factory to complete integration test --- krebs/3modules/buildbot/master.nix | 47 +++++++++++++++++++++++++------------- krebs/5pkgs/krebs-ci/notes | 8 +++---- 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index e66e0d6b2..0d9c53977 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -59,27 +59,28 @@ let ###### The actual build # couple of fast steps: f = util.BuildFactory() + # some slow steps + s = util.BuildFactory() ## fetch repo grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') f.addStep(grab_repo) + s.addStep(grab_repo) # the dependencies which are used by the test script - deps = [ "gnumake", "jq" ] - nixshell = ["nix-shell", "-p" ] + deps + [ "--run" ] + deps = [ "gnumake", "jq", "(import {}).pkgs.krebs-ci" ] + nixshell = ["nix-shell", "-I", ".", "-p" ] + deps + [ "--run" ] + def addShell(f,**kwargs): f.addStep(steps.ShellCommand(**kwargs)) - addShell(f,name="centos7-eval",env={"LOGNAME": "shared", - "get" : "krebs.deploy", - "filter" : "json" - }, - command=nixshell + ["make -s eval system=test-centos7"]) + addShell(f,name="centos7-eval",env={"LOGNAME": "shared"}, + command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"]) + + addShell(f,name="wolf-eval",env={"LOGNAME": "shared"}, + command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"]) - addShell(f,name="wolf-eval",env={"LOGNAME": "shared", - "get" : "krebs.deploy", - "filter" : "json" - }, - command=nixshell + ["make -s eval system=wolf"]) + addShell(f,name="eval-cross-check",env={"LOGNAME": "shared"}, + command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"]) c['builders'] = [] c['builders'].append( @@ -87,11 +88,20 @@ let slavenames=slavenames, factory=f)) - # TODO slow build + # slave needs 2 files: + # * cac.json + # * retiolum + for file in ["cac.json", "retiolum.rsa_key.priv"]: + s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file), + slavedest=file)) + + addShell(s,name="complete-build-centos7",env={"LOGNAME": "shared"}, + command=nix-shell + ["krebs-ci"]) + c['builders'].append( util.BuilderConfig(name="full-tests", slavenames=slavenames, - factory=f)) + factory=s)) ####### Status of Builds c['status'] = [] @@ -119,8 +129,8 @@ let # TODO: multiple channels channels=["${cfg.irc.channel}"], notify_events={ - #'success': 1, - #'failure': 1, + 'success': 1, + 'failure': 1, 'exception': 1, 'successToFailure': 1, 'failureToSuccess': 1, @@ -221,6 +231,7 @@ let path = [ pkgs.git ]; serviceConfig = let workdir="${lib.shell.escape cfg.workDir}"; + secretsdir="${lib.shell.escape (toString )}"; # TODO: check if git is the only dep in { PermissionsStartOnly = true; @@ -236,6 +247,10 @@ let fi # always override the master.cfg cp ${buildbot-master-config} ${workdir}/master.cfg + # copy secrets + cp ${secretsdir}/cac.json ${workdir} + cp ${secretsdir}/retiolum-ci.rsa_key.priv \ + ${workdir}/retiolum.rsa_key.priv # sanity ${buildbot}/bin/buildbot checkconfig ${workdir} diff --git a/krebs/5pkgs/krebs-ci/notes b/krebs/5pkgs/krebs-ci/notes index f6f3da8db..f162656f7 100755 --- a/krebs/5pkgs/krebs-ci/notes +++ b/krebs/5pkgs/krebs-ci/notes @@ -19,10 +19,10 @@ fi krebs_secrets=$(mktemp -d) sec_file=$krebs_secrets/cac_config krebs_ssh=$krebs_secrets/tempssh -cac_resources_cache=$krebs_secrets/res_cache.json -cac_servers_cache=$krebs_secrets/servers_cache.json -cac_tasks_cache=$krebs_secrets/tasks_cache.json -cac_templates_cache=$krebs_secrets/templates_cache.json +export cac_resources_cache=$krebs_secrets/res_cache.json +export cac_servers_cache=$krebs_secrets/servers_cache.json +export cac_tasks_cache=$krebs_secrets/tasks_cache.json +export cac_templates_cache=$krebs_secrets/templates_cache.json # we need to receive this key from buildmaster to speed up tinc bootstrap TRAP="rm -r $krebs_secrets;exit" trap "$TRAP" INT TERM EXIT -- cgit v1.2.3 From dc8e270d2a5346e4316b7c2050b26fd428ec3fc3 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 23 Dec 2015 00:06:27 +0100 Subject: m 3 buildbot: add new slow factory to complete integration test --- krebs/3modules/buildbot/master.nix | 52 +++++++++++++++++++++++++------------- krebs/3modules/buildbot/slave.nix | 1 + krebs/5pkgs/krebs-ci/notes | 8 +++--- 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index e66e0d6b2..b4fd6bb2f 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -59,27 +59,28 @@ let ###### The actual build # couple of fast steps: f = util.BuildFactory() + # some slow steps + s = util.BuildFactory() ## fetch repo grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') f.addStep(grab_repo) + s.addStep(grab_repo) # the dependencies which are used by the test script - deps = [ "gnumake", "jq" ] - nixshell = ["nix-shell", "-p" ] + deps + [ "--run" ] + deps = [ "gnumake", "jq", "(import {}).pkgs.krebs-ci" ] + nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ] + def addShell(f,**kwargs): f.addStep(steps.ShellCommand(**kwargs)) - addShell(f,name="centos7-eval",env={"LOGNAME": "shared", - "get" : "krebs.deploy", - "filter" : "json" - }, - command=nixshell + ["make -s eval system=test-centos7"]) + addShell(f,name="centos7-eval",env={"LOGNAME": "shared"}, + command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"]) + + addShell(f,name="wolf-eval",env={"LOGNAME": "shared"}, + command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"]) - addShell(f,name="wolf-eval",env={"LOGNAME": "shared", - "get" : "krebs.deploy", - "filter" : "json" - }, - command=nixshell + ["make -s eval system=wolf"]) + addShell(f,name="eval-cross-check",env={"LOGNAME": "shared"}, + command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"]) c['builders'] = [] c['builders'].append( @@ -87,11 +88,20 @@ let slavenames=slavenames, factory=f)) - # TODO slow build + # slave needs 2 files: + # * cac.json + # * retiolum + for file in ["cac.json", "retiolum.rsa_key.priv"]: + s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file), + slavedest=file)) + + addShell(s,name="complete-build-centos7",env={"LOGNAME": "shared"}, + command=nixshell + ["krebs-ci"]) + c['builders'].append( util.BuilderConfig(name="full-tests", slavenames=slavenames, - factory=f)) + factory=s)) ####### Status of Builds c['status'] = [] @@ -106,7 +116,7 @@ let forceBuild = 'auth', forceAllBuilds = 'auth', pingBuilder = False, - stopBuild = False, + stopBuild = 'auth', stopAllBuilds = False, cancelPendingBuild = False, ) @@ -119,8 +129,8 @@ let # TODO: multiple channels channels=["${cfg.irc.channel}"], notify_events={ - #'success': 1, - #'failure': 1, + 'success': 1, + 'failure': 1, 'exception': 1, 'successToFailure': 1, 'failureToSuccess': 1, @@ -219,8 +229,12 @@ let after = [ "network.target" ]; wantedBy = [ "multi-user.target" ]; path = [ pkgs.git ]; + environment = { + SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; + }; serviceConfig = let workdir="${lib.shell.escape cfg.workDir}"; + secretsdir="${lib.shell.escape (toString )}"; # TODO: check if git is the only dep in { PermissionsStartOnly = true; @@ -236,6 +250,10 @@ let fi # always override the master.cfg cp ${buildbot-master-config} ${workdir}/master.cfg + # copy secrets + cp ${secretsdir}/cac.json ${workdir} + cp ${secretsdir}/retiolum-ci.rsa_key.priv \ + ${workdir}/retiolum.rsa_key.priv # sanity ${buildbot}/bin/buildbot checkconfig ${workdir} diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix index 65291f63e..8711a287a 100644 --- a/krebs/3modules/buildbot/slave.nix +++ b/krebs/3modules/buildbot/slave.nix @@ -144,6 +144,7 @@ let path = default-packages ++ cfg.packages; environment = { + SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; NIX_REMOTE="daemon"; } // cfg.extraEnviron; diff --git a/krebs/5pkgs/krebs-ci/notes b/krebs/5pkgs/krebs-ci/notes index f6f3da8db..f162656f7 100755 --- a/krebs/5pkgs/krebs-ci/notes +++ b/krebs/5pkgs/krebs-ci/notes @@ -19,10 +19,10 @@ fi krebs_secrets=$(mktemp -d) sec_file=$krebs_secrets/cac_config krebs_ssh=$krebs_secrets/tempssh -cac_resources_cache=$krebs_secrets/res_cache.json -cac_servers_cache=$krebs_secrets/servers_cache.json -cac_tasks_cache=$krebs_secrets/tasks_cache.json -cac_templates_cache=$krebs_secrets/templates_cache.json +export cac_resources_cache=$krebs_secrets/res_cache.json +export cac_servers_cache=$krebs_secrets/servers_cache.json +export cac_tasks_cache=$krebs_secrets/tasks_cache.json +export cac_templates_cache=$krebs_secrets/templates_cache.json # we need to receive this key from buildmaster to speed up tinc bootstrap TRAP="rm -r $krebs_secrets;exit" trap "$TRAP" INT TERM EXIT -- cgit v1.2.3 From 14ddb767eb10dbe43d3112c4b4674f6c1d4ff32a Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 23 Dec 2015 11:18:00 +0100 Subject: k 5 mv krebs-ci test/infest-cac-centos7 --- krebs/3modules/buildbot/master.nix | 6 +- krebs/5pkgs/default.nix | 4 + krebs/5pkgs/krebs-ci/default.nix | 37 -------- krebs/5pkgs/krebs-ci/notes | 116 ----------------------- krebs/5pkgs/test/infest-cac-centos7/default.nix | 39 ++++++++ krebs/5pkgs/test/infest-cac-centos7/notes | 117 ++++++++++++++++++++++++ 6 files changed, 163 insertions(+), 156 deletions(-) delete mode 100644 krebs/5pkgs/krebs-ci/default.nix delete mode 100755 krebs/5pkgs/krebs-ci/notes create mode 100644 krebs/5pkgs/test/infest-cac-centos7/default.nix create mode 100755 krebs/5pkgs/test/infest-cac-centos7/notes diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index b4fd6bb2f..483ba18e7 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -67,7 +67,7 @@ let s.addStep(grab_repo) # the dependencies which are used by the test script - deps = [ "gnumake", "jq", "(import {}).pkgs.krebs-ci" ] + deps = [ "gnumake", "jq", "(import {}).pkgs.test.infest-cac-centos7" ] nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ] def addShell(f,**kwargs): @@ -95,8 +95,8 @@ let s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file), slavedest=file)) - addShell(s,name="complete-build-centos7",env={"LOGNAME": "shared"}, - command=nixshell + ["krebs-ci"]) + addShell(s,name="infest-cac-centos7",env={"LOGNAME": "shared"}, + command=nixshell + ["infest-cac-centos7"]) c['builders'].append( util.BuilderConfig(name="full-tests", diff --git a/krebs/5pkgs/default.nix b/krebs/5pkgs/default.nix index 7df7b7d3c..0562fe836 100644 --- a/krebs/5pkgs/default.nix +++ b/krebs/5pkgs/default.nix @@ -40,6 +40,10 @@ subdirs // rec { } ''; + test = { + infest-cac-centos7 = pkgs.callPackage ./test/infest-cac-centos7 {}; + }; + execveBin = name: cfg: execve name (cfg // { destination = "/bin/${name}"; }); writeC = name: { destination ? "" }: src: pkgs.runCommand name {} '' diff --git a/krebs/5pkgs/krebs-ci/default.nix b/krebs/5pkgs/krebs-ci/default.nix deleted file mode 100644 index f5b302b52..000000000 --- a/krebs/5pkgs/krebs-ci/default.nix +++ /dev/null @@ -1,37 +0,0 @@ -{ stdenv, coreutils,makeWrapper, cac, cacpanel, gnumake, gnused, jq, openssh, ... }: - -stdenv.mkDerivation rec { - name = "krebs-ci-0.1.0"; - - src = ./notes; - - phases = [ - "installPhase" - ]; - buildInputs = [ makeWrapper ]; - - path = stdenv.lib.makeSearchPath "bin" [ - coreutils - cac - cacpanel - gnumake - gnused - jq - openssh - ]; - - installPhase = - '' - mkdir -p $out/bin - cp ${src} $out/bin/krebs-ci - chmod +x $out/bin/krebs-ci - wrapProgram $out/bin/krebs-ci \ - --prefix PATH : ${path} - ''; - meta = with stdenv.lib; { - homepage = http://krebsco.de; - description = "Krebs CI Scripts"; - license = licenses.wtfpl; - maintainers = [ maintainers.makefu ]; - }; -} diff --git a/krebs/5pkgs/krebs-ci/notes b/krebs/5pkgs/krebs-ci/notes deleted file mode 100755 index f162656f7..000000000 --- a/krebs/5pkgs/krebs-ci/notes +++ /dev/null @@ -1,116 +0,0 @@ -#! /bin/sh - -# nix-shell -p gnumake jq openssh cac cacpanel -set -eufx - -# 2 secrets are required: - -krebs_cred=${krebs_cred-./cac.json} -retiolum_key=${retiolum_key-./retiolum.rsa_key.priv} - -# Sanity -if test ! -r "$krebs_cred";then - echo "\$krebs_cred=$krebs_cred must be readable"; exit 1 -fi -if test ! -r "$retiolum_key";then - echo "\$retiolum_key=$retiolum_key must be readable"; exit 1 -fi - -krebs_secrets=$(mktemp -d) -sec_file=$krebs_secrets/cac_config -krebs_ssh=$krebs_secrets/tempssh -export cac_resources_cache=$krebs_secrets/res_cache.json -export cac_servers_cache=$krebs_secrets/servers_cache.json -export cac_tasks_cache=$krebs_secrets/tasks_cache.json -export cac_templates_cache=$krebs_secrets/templates_cache.json -# we need to receive this key from buildmaster to speed up tinc bootstrap -TRAP="rm -r $krebs_secrets;exit" -trap "$TRAP" INT TERM EXIT - -cat > $sec_file <&1\ - | jq -r .servername) - -id=servername:$name -trap "cac delete $id;$TRAP" INT TERM EXIT -# TODO: timeout? -# cac_always_update=true cac waitstatus $id "Powered On" - -wait_login_cac(){ - # timeout - for t in `seq 180`;do - # now we have a working cac server - if cac ssh $1 cat /etc/redhat-release | \ - grep CentOS ;then - return 0 - fi - sleep 10 - done - return 1 -} -# die on timeout -wait_login_cac $id - -mkdir -p shared/2configs/temp -cac generatenetworking $id > \ - shared/2configs/temp/networking.nix -# new temporary ssh key we will use to log in after infest -ssh-keygen -f $krebs_ssh -N "" -cp $retiolum_key $krebs_secrets/retiolum.rsa_key.priv -# we override the directories for secrets and stockholm -# additionally we set the ssh key we generated -ip=$(cac getserver $id | jq -r .ip) - -cat > shared/2configs/temp/dirs.nix < $krebs_secrets/infest -sh -x $krebs_secrets/infest - -# TODO: generate secrets directory $krebs_secrets for nix import -cac powerop $id reset - -wait_login(){ - # timeout - for t in `seq 20`;do - # now we have a working cac server - if ssh -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -i $krebs_ssh \ - -o ConnectTimeout=10 \ - -o BatchMode=yes \ - root@$1 nixos-version ;then - return 0 - fi - sleep 10 - done - return 1 -} -wait_login $ip diff --git a/krebs/5pkgs/test/infest-cac-centos7/default.nix b/krebs/5pkgs/test/infest-cac-centos7/default.nix new file mode 100644 index 000000000..7f2e3f231 --- /dev/null +++ b/krebs/5pkgs/test/infest-cac-centos7/default.nix @@ -0,0 +1,39 @@ +{ stdenv, coreutils,makeWrapper, cac, cacpanel, gnumake, gnused, jq, openssh, ... }: + +stdenv.mkDerivation rec { + name = "${shortname}-${version}"; + shortname = "infest-cac-centos7"; + version = "0.2.0"; + + src = ./notes; + + phases = [ + "installPhase" + ]; + buildInputs = [ makeWrapper ]; + + path = stdenv.lib.makeSearchPath "bin" [ + coreutils + cac + cacpanel + gnumake + gnused + jq + openssh + ]; + + installPhase = + '' + mkdir -p $out/bin + cp ${src} $out/bin/${shortname} + chmod +x $out/bin/${shortname} + wrapProgram $out/bin/${shortname} \ + --prefix PATH : ${path} + ''; + meta = with stdenv.lib; { + homepage = http://krebsco.de; + description = "Krebs CI Scripts"; + license = licenses.wtfpl; + maintainers = [ maintainers.makefu ]; + }; +} diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes new file mode 100755 index 000000000..1e350084c --- /dev/null +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -0,0 +1,117 @@ +#! /bin/sh + +# nix-shell -p gnumake jq openssh cac cacpanel +set -eufx + +# 2 secrets are required: + +krebs_cred=${krebs_cred-./cac.json} +retiolum_key=${retiolum_key-./retiolum.rsa_key.priv} + +# Sanity +if test ! -r "$krebs_cred";then + echo "\$krebs_cred=$krebs_cred must be readable"; exit 1 +fi +if test ! -r "$retiolum_key";then + echo "\$retiolum_key=$retiolum_key must be readable"; exit 1 +fi + +krebs_secrets=$(mktemp -d) +sec_file=$krebs_secrets/cac_config +krebs_ssh=$krebs_secrets/tempssh +export cac_resources_cache=$krebs_secrets/res_cache.json +export cac_servers_cache=$krebs_secrets/servers_cache.json +export cac_tasks_cache=$krebs_secrets/tasks_cache.json +export cac_templates_cache=$krebs_secrets/templates_cache.json +# we need to receive this key from buildmaster to speed up tinc bootstrap +TRAP="rm -r $krebs_secrets;trap - INT TERM EXIT" +trap "$TRAP" INT TERM EXIT + +cat > $sec_file <&1\ + | jq -r .servername) + +id=servername:$name +trap "cac delete $id;$TRAP;exit" INT TERM EXIT +# TODO: timeout? + +wait_login_cac(){ + # timeout + for t in `seq 180`;do + # now we have a working cac server + if cac ssh $1 -o ConnectTimeout=10 \ + -o BatchMode=yes \ + cat /etc/redhat-release | \ + grep CentOS ;then + return 0 + fi + sleep 10 + done + return 1 +} +# die on timeout +wait_login_cac $id + +mkdir -p shared/2configs/temp +cac generatenetworking $id > \ + shared/2configs/temp/networking.nix +# new temporary ssh key we will use to log in after infest +ssh-keygen -f $krebs_ssh -N "" +cp $retiolum_key $krebs_secrets/retiolum.rsa_key.priv +# we override the directories for secrets and stockholm +# additionally we set the ssh key we generated +ip=$(cac getserver $id | jq -r .ip) + +cat > shared/2configs/temp/dirs.nix < $krebs_secrets/infest +sh -x $krebs_secrets/infest + +# TODO: generate secrets directory $krebs_secrets for nix import +cac powerop $id reset + +wait_login(){ + # timeout + for t in `seq 20`;do + # now we have a working cac server + if ssh -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -i $krebs_ssh \ + -o ConnectTimeout=10 \ + -o BatchMode=yes \ + root@$1 nixos-version ;then + return 0 + fi + sleep 10 + done + return 1 +} +wait_login $ip -- cgit v1.2.3 From cf3391704d88c49afba652715e1153888bf46099 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 23 Dec 2015 16:02:58 +0100 Subject: k 5 test/infest*: remove batch mode from cac ssh call this leads to "permission denied" --- krebs/5pkgs/test/infest-cac-centos7/notes | 1 - 1 file changed, 1 deletion(-) diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes index 1e350084c..5fd0cae61 100755 --- a/krebs/5pkgs/test/infest-cac-centos7/notes +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -53,7 +53,6 @@ wait_login_cac(){ for t in `seq 180`;do # now we have a working cac server if cac ssh $1 -o ConnectTimeout=10 \ - -o BatchMode=yes \ cat /etc/redhat-release | \ grep CentOS ;then return 0 -- cgit v1.2.3 From 9a386718714d70f7100b5de297dfd0869d98e47b Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 23 Dec 2015 16:06:41 +0100 Subject: k 3 buildbot: fix merge fuckup --- krebs/3modules/buildbot/master.nix | 5 ----- 1 file changed, 5 deletions(-) diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index 19aecead1..483ba18e7 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -95,13 +95,8 @@ let s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file), slavedest=file)) -<<<<<<< HEAD addShell(s,name="infest-cac-centos7",env={"LOGNAME": "shared"}, command=nixshell + ["infest-cac-centos7"]) -======= - addShell(s,name="complete-build-centos7",env={"LOGNAME": "shared"}, - command=nix-shell + ["krebs-ci"]) ->>>>>>> f59080e76f950a5a8e33d1edd4314ffaa14187fc c['builders'].append( util.BuilderConfig(name="full-tests", -- cgit v1.2.3 From 1ef9af2c9a49490a2dda21884ad761675c520d1a Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 23 Dec 2015 16:20:27 +0100 Subject: k 3 buildbot/master: send sigterm before sigkill for cleanup --- krebs/3modules/buildbot/master.nix | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index 483ba18e7..6ce708769 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -95,8 +95,10 @@ let s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file), slavedest=file)) - addShell(s,name="infest-cac-centos7",env={"LOGNAME": "shared"}, - command=nixshell + ["infest-cac-centos7"]) + addShell(s, name="infest-cac-centos7",env={"LOGNAME": "shared"}, + sigtermTime=60, # SIGTERM 1 minute before SIGKILL + timeout=5400, # 1.5h timeout + command=nixshell + ["infest-cac-centos7"]) c['builders'].append( util.BuilderConfig(name="full-tests", -- cgit v1.2.3 From 6b7506dc672b4bd658088bf37fad06fd64c777fe Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 23 Dec 2015 16:37:02 +0100 Subject: k 3 buildbot: add rsync as explicit dep do not be pure yet --- krebs/3modules/buildbot/master.nix | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index 6ce708769..6bf3fda2c 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -67,19 +67,24 @@ let s.addStep(grab_repo) # the dependencies which are used by the test script - deps = [ "gnumake", "jq", "(import {}).pkgs.test.infest-cac-centos7" ] + deps = [ "gnumake", "jq","nix", + "(import {}).pkgs.test.infest-cac-centos7", + "rsync" ] + # TODO: --pure , prepare ENV in nix-shell command: + # SSL_CERT_FILE,LOGNAME,NIX_REMOTE nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ] - + env = {"LOGNAME": "shared", + "NIX_REMOTE": "daemon"} def addShell(f,**kwargs): f.addStep(steps.ShellCommand(**kwargs)) - addShell(f,name="centos7-eval",env={"LOGNAME": "shared"}, + addShell(f,name="centos7-eval",env=env, command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"]) - addShell(f,name="wolf-eval",env={"LOGNAME": "shared"}, + addShell(f,name="wolf-eval",env=env, command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"]) - addShell(f,name="eval-cross-check",env={"LOGNAME": "shared"}, + addShell(f,name="eval-cross-check",env=env, command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"]) c['builders'] = [] @@ -95,7 +100,7 @@ let s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file), slavedest=file)) - addShell(s, name="infest-cac-centos7",env={"LOGNAME": "shared"}, + addShell(s, name="infest-cac-centos7",env=env, sigtermTime=60, # SIGTERM 1 minute before SIGKILL timeout=5400, # 1.5h timeout command=nixshell + ["infest-cac-centos7"]) -- cgit v1.2.3 From 3adf78473d2deff0b991d7222e928fa2888529f6 Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 24 Dec 2015 00:02:59 +0100 Subject: k 3 buildbot.master: refactor see buildbot-standalone.nix in shared/2configs for the current buildbot config --- krebs/3modules/buildbot/master.nix | 341 ++++++++++++++++++++------------ shared/2configs/buildbot-standalone.nix | 107 +++++++++- 2 files changed, 317 insertions(+), 131 deletions(-) diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index 6bf3fda2c..7078000fe 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -7,134 +7,81 @@ let # -*- python -*- from buildbot.plugins import * import re - + import json c = BuildmasterConfig = {} c['slaves'] = [] - # TODO: template potential buildslaves - # TODO: set password? - slavenames= [ 'testslave' ] - for i in slavenames: - c['slaves'].append(buildslave.BuildSlave(i, "krebspass")) + slaves = json.loads('${builtins.toJSON cfg.slaves}') + slavenames = [ s for s in slaves ] + for k,v in slaves.items(): + c['slaves'].append(buildslave.BuildSlave(k, v)) + # TODO: configure protocols? c['protocols'] = {'pb': {'port': 9989}} ####### Build Inputs - stockholm_repo = 'http://cgit.gum/stockholm' - c['change_source'] = [] - c['change_source'].append(changes.GitPoller( - stockholm_repo, - workdir='stockholm-poller', branch='master', - project='stockholm', - pollinterval=120)) + c['change_source'] = cs = [] + + ${ concatStringsSep "\n" + (mapAttrsToList (n: v: '' + #### Change_Source: Begin of ${n} + ${v} + #### Change_Source: End of ${n} + '') cfg.change_source )} ####### Build Scheduler - # TODO: configure scheduler - c['schedulers'] = [] - - # test the master real quick - fast = schedulers.SingleBranchScheduler( - change_filter=util.ChangeFilter(branch="master"), - name="fast-master-test", - builderNames=["fast-tests"]) - - force = schedulers.ForceScheduler( - name="force", - builderNames=["full-tests"]) - - # files everyone depends on or are part of the share branch - def shared_files(change): - r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)") - for file in change.files: - if r.match(file): - return True - return False - - full = schedulers.SingleBranchScheduler( - change_filter=util.ChangeFilter(branch="master"), - fileIsImportant=shared_files, - name="full-master-test", - builderNames=["full-tests"]) - c['schedulers'] = [ fast, force, full ] - ###### The actual build - # couple of fast steps: - f = util.BuildFactory() - # some slow steps - s = util.BuildFactory() - ## fetch repo - grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') - f.addStep(grab_repo) - s.addStep(grab_repo) - - # the dependencies which are used by the test script - deps = [ "gnumake", "jq","nix", - "(import {}).pkgs.test.infest-cac-centos7", - "rsync" ] - # TODO: --pure , prepare ENV in nix-shell command: - # SSL_CERT_FILE,LOGNAME,NIX_REMOTE - nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ] - env = {"LOGNAME": "shared", - "NIX_REMOTE": "daemon"} - def addShell(f,**kwargs): - f.addStep(steps.ShellCommand(**kwargs)) - - addShell(f,name="centos7-eval",env=env, - command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"]) - - addShell(f,name="wolf-eval",env=env, - command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"]) - - addShell(f,name="eval-cross-check",env=env, - command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"]) - - c['builders'] = [] - c['builders'].append( - util.BuilderConfig(name="fast-tests", - slavenames=slavenames, - factory=f)) - - # slave needs 2 files: - # * cac.json - # * retiolum - for file in ["cac.json", "retiolum.rsa_key.priv"]: - s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file), - slavedest=file)) - - addShell(s, name="infest-cac-centos7",env=env, - sigtermTime=60, # SIGTERM 1 minute before SIGKILL - timeout=5400, # 1.5h timeout - command=nixshell + ["infest-cac-centos7"]) - - c['builders'].append( - util.BuilderConfig(name="full-tests", - slavenames=slavenames, - factory=s)) - - ####### Status of Builds - c['status'] = [] - - from buildbot.status import html - from buildbot.status.web import authz, auth - # TODO: configure if http is wanted - authz_cfg=authz.Authz( - # TODO: configure user/pw - auth=auth.BasicAuth([("krebs","bob")]), - gracefulShutdown = False, - forceBuild = 'auth', - forceAllBuilds = 'auth', - pingBuilder = False, - stopBuild = 'auth', - stopAllBuilds = False, - cancelPendingBuild = False, - ) - # TODO: configure nginx - c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg)) - - from buildbot.status import words + c['schedulers'] = sched = [] + + ${ concatStringsSep "\n" + (mapAttrsToList (n: v: '' + #### Schedulers: Begin of ${n} + ${v} + #### Schedulers: End of ${n} + '') cfg.scheduler )} + + ###### Builder + c['builders'] = bu = [] + + # Builder Pre: Begin + ${cfg.builder_pre} + # Builder Pre: End + + ${ concatStringsSep "\n" + (mapAttrsToList (n: v: '' + #### Builder: Begin of ${n} + ${v} + #### Builder: End of ${n} + '') cfg.builder )} + + + ####### Status + c['status'] = st = [] + + # If you want to configure this url, override with extraConfig + c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/" + + ${optionalString (cfg.web.enable) '' + from buildbot.status import html + from buildbot.status.web import authz, auth + authz_cfg=authz.Authz( + auth=auth.BasicAuth([ ("${cfg.web.username}","${cfg.web.password}") ]), + # TODO: configure harder + gracefulShutdown = False, + forceBuild = 'auth', + forceAllBuilds = 'auth', + pingBuilder = False, + stopBuild = 'auth', + stopAllBuilds = 'auth', + cancelPendingBuild = 'auth' + ) + # TODO: configure krebs.nginx + st.append(html.WebStatus(http_port=${toString cfg.web.port}, authz=authz_cfg)) + ''} + ${optionalString (cfg.irc.enable) '' - irc = words.IRC("${cfg.irc.server}", "krebsbuild", - # TODO: multiple channels - channels=["${cfg.irc.channel}"], + from buildbot.status import words + irc = words.IRC("${cfg.irc.server}", "${cfg.irc.nick}", + channels=${builtins.toJSON cfg.irc.channels}, notify_events={ 'success': 1, 'failure': 1, @@ -145,15 +92,20 @@ let c['status'].append(irc) ''} + ${ concatStringsSep "\n" + (mapAttrsToList (n: v: '' + #### Status: Begin of ${n} + ${v} + #### Status: End of ${n} + '') cfg.status )} + ####### PROJECT IDENTITY - c['title'] = "Stockholm" + c['title'] = "${cfg.title}" c['titleURL'] = "http://krebsco.de" - #c['buildbotURL'] = "http://buildbot.krebsco.de/" - # TODO: configure url - c['buildbotURL'] = "http://vbob:8010/" ####### DB URL + # TODO: configure c['db'] = { 'db_url' : "sqlite:///state.sqlite", } @@ -164,6 +116,13 @@ let api = { enable = mkEnableOption "Buildbot Master"; + title = mkOption { + default = "Buildbot CI"; + type = types.str; + description = '' + Title of the Buildbot Installation + ''; + }; workDir = mkOption { default = "/var/lib/buildbot/master"; type = types.str; @@ -172,16 +131,144 @@ let Will be created on startup. ''; }; + + slaves = mkOption { + default = {}; + type = types.attrsOf types.str; + description = '' + Attrset of slavenames with their passwords + slavename = slavepassword + ''; + }; + + change_source = mkOption { + default = {}; + type = types.attrsOf types.str; + example = { + stockholm = '' + cs.append(changes.GitPoller( + 'http://cgit.gum/stockholm', + workdir='stockholm-poller', branch='master', + project='stockholm', + pollinterval=120)) + ''; + }; + description = '' + Attrset of all the change_sources which should be configured. + It will be directly included into the master configuration. + + At the end an change object should be appended to cs + ''; + }; + + scheduler = mkOption { + default = {}; + type = types.attrsOf types.str; + example = { + force-scheduler = '' + sched.append(schedulers.ForceScheduler( + name="force", + builderNames=["full-tests"])) + ''; + }; + description = '' + Attrset of all the schedulers which should be configured. + It will be directly included into the master configuration. + + At the end an change object should be appended to sched + ''; + }; + + builder_pre = mkOption { + default = ""; + type = types.lines; + example = '' + grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') + ''; + description = '' + some code before the builders are being assembled. + can be used to define functions used by multiple builders + ''; + }; + + builder = mkOption { + default = {}; + type = types.attrsOf types.str; + example = { + fast-test = '' + ''; + }; + description = '' + Attrset of all the builder which should be configured. + It will be directly included into the master configuration. + + At the end an change object should be appended to bu + ''; + }; + + status = mkOption { + default = {}; + type = types.attrsOf types.str; + description = '' + Attrset of all the extra status which should be configured. + It will be directly included into the master configuration. + + At the end an change object should be appended to st + + Right now IRC and Web status can be configured by setting + buildbot.master.irc.enable and + buildbot.master.web.enable + ''; + }; + + # Configurable Stati + web = mkOption { + default = {}; + type = types.submodule ({ config2, ... }: { + options = { + enable = mkEnableOption "Buildbot Master Web Status"; + username = mkOption { + default = "krebs"; + type = types.str; + description = '' + username for web authentication + ''; + }; + hostname = mkOption { + default = config.networking.hostName; + type = types.str; + description = '' + web interface Hostname + ''; + }; + password = mkOption { + default = "bob"; + type = types.str; + description = '' + password for web authentication + ''; + }; + port = mkOption { + default = 8010; + type = types.int; + description = '' + port for buildbot web status + ''; + }; + }; + }); + }; + irc = mkOption { default = {}; type = types.submodule ({ config, ... }: { options = { enable = mkEnableOption "Buildbot Master IRC Status"; - channel = mkOption { - default = "nix-buildbot-meetup"; - type = types.str; + channels = mkOption { + default = [ "nix-buildbot-meetup" ]; + type = with types; listOf str; description = '' - irc channel the bot should connect to + irc channels the bot should connect to ''; }; allowForce = mkOption { @@ -235,6 +322,7 @@ let description = "Buildbot Master"; after = [ "network.target" ]; wantedBy = [ "multi-user.target" ]; + # TODO: add extra dependencies to master like svn and cvs path = [ pkgs.git ]; environment = { SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; @@ -242,7 +330,6 @@ let serviceConfig = let workdir="${lib.shell.escape cfg.workDir}"; secretsdir="${lib.shell.escape (toString )}"; - # TODO: check if git is the only dep in { PermissionsStartOnly = true; Type = "forking"; diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix index adf44cada..baab059c9 100644 --- a/shared/2configs/buildbot-standalone.nix +++ b/shared/2configs/buildbot-standalone.nix @@ -8,16 +8,115 @@ in { }; networking.firewall.allowedTCPPorts = [ 8010 ]; krebs.buildbot.master = { + slaves = { + testslave = "krebspass"; + testslave2 = "krebspass"; + }; + change_source.stockholm = '' + stockholm_repo = 'http://cgit.gum/stockholm' + cs.append(changes.GitPoller( + stockholm_repo, + workdir='stockholm-poller', branch='master', + project='stockholm', + pollinterval=120)) + ''; + scheduler = { + force-scheduler = '' + sched.append(schedulers.ForceScheduler( + name="force", + builderNames=["full-tests"])) + ''; + fast-tests-scheduler = '' + # test the master real quick + sched.append(schedulers.SingleBranchScheduler( + change_filter=util.ChangeFilter(branch="master"), + name="fast-master-test", + builderNames=["fast-tests"])) + ''; + full-master-scheduler = '' + # files everyone depends on or are part of the share branch + def shared_files(change): + r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)") + for file in change.files: + if r.match(file): + return True + return False + + sched.append(schedulers.SingleBranchScheduler( + change_filter=util.ChangeFilter(branch="master"), + fileIsImportant=shared_files, + name="full-master-test", + builderNames=["full-tests"])) + ''; + }; + builder_pre = '' + # prepare grab_repo step for stockholm + stockholm_repo = "http://cgit.gum.retiolum/stockholm" + grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') + + env = {"LOGNAME": "shared", "NIX_REMOTE": "daemon"} + + # prepare nix-shell + # the dependencies which are used by the test script + deps = [ "gnumake", "jq","nix","rsync", + "(import {}).pkgs.test.infest-cac-centos7" ] + # TODO: --pure , prepare ENV in nix-shell command: + # SSL_CERT_FILE,LOGNAME,NIX_REMOTE + nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ] + + # prepare addShell function + def addShell(factory,**kwargs): + factory.addStep(steps.ShellCommand(**kwargs)) + ''; + builder = { + fast-tests = '' + f = util.BuildFactory() + f.addStep(grab_repo) + addShell(f,name="centos7-eval",env=env, + command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"]) + + addShell(f,name="wolf-eval",env=env, + command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"]) + + addShell(f,name="eval-cross-check",env=env, + command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"]) + + bu.append(util.BuilderConfig(name="fast-tests", + slavenames=slavenames, + factory=f)) + ''; + slow-tests = '' + s = util.BuildFactory() + s.addStep(grab_repo) + + # slave needs 2 files: + # * cac.json + # * retiolum + for file in ["cac.json", "retiolum.rsa_key.priv"]: + s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/{}".format(file), + slavedest=file)) + + addShell(s, name="infest-cac-centos7",env=env, + sigtermTime=60, # SIGTERM 1 minute before SIGKILL + timeout=5400, # 1.5h timeout + command=nixshell + ["infest-cac-centos7"]) + + bu.append(util.BuilderConfig(name="full-tests", + slavenames=slavenames, + factory=s)) + ''; + }; enable = true; + web = { + enable = true; + }; irc = { enable = true; + nick = "shared-buildbot"; server = "cd.retiolum"; - channel = "retiolum"; + channels = [ "retiolum" ]; allowForce = true; }; - extraConfig = '' - c['buildbotURL'] = "http://${config.krebs.build.host.name}:8010/" - ''; }; krebs.buildbot.slave = { -- cgit v1.2.3 From 6e4351044195f1f3b5708785e760b9d118e2c229 Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 24 Dec 2015 02:20:24 +0100 Subject: k 5 test/infest*: up limit of final connect --- krebs/5pkgs/test/infest-cac-centos7/notes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes index 5fd0cae61..5bb5de2c4 100755 --- a/krebs/5pkgs/test/infest-cac-centos7/notes +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -99,7 +99,7 @@ cac powerop $id reset wait_login(){ # timeout - for t in `seq 20`;do + for t in `seq 90`;do # now we have a working cac server if ssh -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ -- cgit v1.2.3 From 9ae664209328f6030bf3773e09dce7bcd14e82b4 Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 24 Dec 2015 20:37:04 +0100 Subject: k 5 cacpanel: bump version to 0.2.3 --- krebs/5pkgs/cacpanel/default.nix | 4 ++-- krebs/5pkgs/test/infest-cac-centos7/notes | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/krebs/5pkgs/cacpanel/default.nix b/krebs/5pkgs/cacpanel/default.nix index 3e3e2e1fc..3df4dffed 100644 --- a/krebs/5pkgs/cacpanel/default.nix +++ b/krebs/5pkgs/cacpanel/default.nix @@ -2,11 +2,11 @@ python3Packages.buildPythonPackage rec { name = "cacpanel-${version}"; - version = "0.2.1"; + version = "0.2.3"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/source/c/cacpanel/cacpanel-${version}.tar.gz"; - sha256 = "1zaazg5r10kgva32zh4fhpw6l6h51ijkwpa322na0kh4x6f6aqj3"; + sha256 = "1fib7416qqv8yzrj75kxra7ccpz9abqh58b6gkaavws2fa6m3mm8"; }; propagatedBuildInputs = with python3Packages; [ diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes index 5bb5de2c4..cfb074423 100755 --- a/krebs/5pkgs/test/infest-cac-centos7/notes +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -29,11 +29,11 @@ trap "$TRAP" INT TERM EXIT cat > $sec_file < Date: Thu, 24 Dec 2015 20:50:23 +0100 Subject: s 2 buildbot: add treestabletimer --- shared/2configs/base.nix | 2 ++ shared/2configs/buildbot-standalone.nix | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/shared/2configs/base.nix b/shared/2configs/base.nix index 0ce336558..4d509d7a6 100644 --- a/shared/2configs/base.nix +++ b/shared/2configs/base.nix @@ -19,6 +19,7 @@ with lib; git.nixpkgs = { url = https://github.com/NixOS/nixpkgs; rev = "6d31e9b81dcd4ab927bb3dc91b612dd5abfa2f80"; + target-path = "/var/src/nixpkgs"; }; dir.secrets = { host = config.krebs.current.host; @@ -27,6 +28,7 @@ with lib; dir.stockholm = { host = config.krebs.current.host; path = mkDefault "${getEnv "HOME"}/stockholm"; + target-path = "/var/src/stockholm"; }; }; diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix index baab059c9..51c600329 100644 --- a/shared/2configs/buildbot-standalone.nix +++ b/shared/2configs/buildbot-standalone.nix @@ -6,11 +6,11 @@ in { buildbot = pkgs-unst.buildbot; buildbot-slave = pkgs-unst.buildbot-slave; }; - networking.firewall.allowedTCPPorts = [ 8010 ]; + networking.firewall.allowedTCPPorts = [ 8010 9989 ]; krebs.buildbot.master = { slaves = { testslave = "krebspass"; - testslave2 = "krebspass"; + omo = "krebspass"; }; change_source.stockholm = '' stockholm_repo = 'http://cgit.gum/stockholm' @@ -33,7 +33,7 @@ in { name="fast-master-test", builderNames=["fast-tests"])) ''; - full-master-scheduler = '' + test-cac-infest-master = '' # files everyone depends on or are part of the share branch def shared_files(change): r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)") @@ -45,6 +45,7 @@ in { sched.append(schedulers.SingleBranchScheduler( change_filter=util.ChangeFilter(branch="master"), fileIsImportant=shared_files, + treeStableTimer=60*60, # master was stable for the last hour name="full-master-test", builderNames=["full-tests"])) ''; -- cgit v1.2.3 From 7f9f7a0cf65212618fbe3fcd85291868b571fae2 Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 24 Dec 2015 20:50:53 +0100 Subject: m 2 urlwatch: add cvs2svn to watchlist --- makefu/2configs/urlwatch.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefu/2configs/urlwatch.nix b/makefu/2configs/urlwatch.nix index cd05f0114..eadffa7dd 100644 --- a/makefu/2configs/urlwatch.nix +++ b/makefu/2configs/urlwatch.nix @@ -12,7 +12,7 @@ http://git.sysphere.org/vicious/log/?qt=grep&q=Next+release https://pypi.python.org/simple/bepasty/ https://pypi.python.org/simple/xstatic/ - + http://cvs2svn.tigris.org/servlets/ProjectDocumentList?folderID=2976 ]; }; } -- cgit v1.2.3 From 58f37bde831877e467646d283b88c17251b84b7c Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 24 Dec 2015 20:51:58 +0100 Subject: m 1 gum: enable urlwatch service --- makefu/1systems/gum.nix | 3 +++ makefu/1systems/pnp.nix | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/makefu/1systems/gum.nix b/makefu/1systems/gum.nix index 417a020fa..93fb3dc3a 100644 --- a/makefu/1systems/gum.nix +++ b/makefu/1systems/gum.nix @@ -15,6 +15,9 @@ in { ../2configs/git/cgit-retiolum.nix ../2configs/mattermost-docker.nix ../2configs/nginx/euer.test.nix + + ../2configs/exim-retiolum.nix + ../2configs/urlwatch.nix ]; diff --git a/makefu/1systems/pnp.nix b/makefu/1systems/pnp.nix index 161bfa3e9..a1b73c0c9 100644 --- a/makefu/1systems/pnp.nix +++ b/makefu/1systems/pnp.nix @@ -28,9 +28,6 @@ ../2configs/Reaktor/titlebot.nix ../2configs/Reaktor/shack-correct.nix - ../2configs/exim-retiolum.nix - ../2configs/urlwatch.nix - # ../2configs/graphite-standalone.nix ]; krebs.urlwatch.verbose = true; -- cgit v1.2.3 From 70264d1e46dc17391f0a3a590ba0749d0a93eda2 Mon Sep 17 00:00:00 2001 From: makefu Date: Fri, 25 Dec 2015 00:04:52 +0100 Subject: k 5 Reaktor: init plugin infrastructure --- krebs/5pkgs/Reaktor/plugins.nix | 38 +++++++++++++++++++++++++++++ krebs/5pkgs/Reaktor/scripts/random-emoji.sh | 6 +++++ krebs/5pkgs/default.nix | 2 ++ 3 files changed, 46 insertions(+) create mode 100644 krebs/5pkgs/Reaktor/plugins.nix create mode 100644 krebs/5pkgs/Reaktor/scripts/random-emoji.sh diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix new file mode 100644 index 000000000..05ede38e1 --- /dev/null +++ b/krebs/5pkgs/Reaktor/plugins.nix @@ -0,0 +1,38 @@ +{ stdenv, lib, pkgs, makeWrapper }: + +rec { + buildReaktorPlugin = { name + # TODO: profiles + , extraConfig + , phases ? [] + , ... } @ attrs: + stdenv.mkDerivation (attrs // { + name = "Reaktor-plugin-" + name; + phases = phases ++ [ "installPhase" ]; + isReaktorPlugin = true; + }); + + random-emoji = buildReaktorPlugin rec { + name = "random-emoji"; + src = ./scripts/random-emoji.sh; + phases = [ "installPhase" ]; + buildInputs = [ makeWrapper ]; + installPhase = '' + mkdir -p $out/bin + install -vm 755 ${src} $out/bin/random-emoji.sh + wrapProgram $out/bin/random-emoji.sh \ + --prefix PATH : ${lib.makeSearchPath "bin" (with pkgs; [ + coreutils + gnused + gnugrep + xmlstarlet + curl])}; + ''; + extraConfig = '' + public_commands.insert(0,{ + 'capname' : "emoji", + 'pattern' : indirect_pattern.format("emoji"), + 'argv' : ["random-emoji.sh"]) + ''; + }; +} diff --git a/krebs/5pkgs/Reaktor/scripts/random-emoji.sh b/krebs/5pkgs/Reaktor/scripts/random-emoji.sh new file mode 100644 index 000000000..386aa68b9 --- /dev/null +++ b/krebs/5pkgs/Reaktor/scripts/random-emoji.sh @@ -0,0 +1,6 @@ +#!/bin/sh +curl http://emojicons.com/random -s | \ + grep data-text | \ + sed -n 's/.*>\(.*\)<\/textarea>/\1/p' | \ + head -n 1 | \ + xmlstarlet unesc diff --git a/krebs/5pkgs/default.nix b/krebs/5pkgs/default.nix index 0562fe836..c4b1dafe4 100644 --- a/krebs/5pkgs/default.nix +++ b/krebs/5pkgs/default.nix @@ -26,6 +26,8 @@ subdirs // rec { inherit (subdirs) get jq; }; + ReaktorPlugins = pkgs.callPackage ./Reaktor/plugins.nix {}; + execve = name: { filename, argv, envp ? {}, destination ? "" }: writeC name { inherit destination; } '' #include -- cgit v1.2.3 From f55b44eb7cffbe0934785afd3a36001ba0713ad1 Mon Sep 17 00:00:00 2001 From: lassulus Date: Sat, 26 Dec 2015 10:43:15 +0100 Subject: l 1 mors: add wordpress/owncloud test --- lass/1systems/mors.nix | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/lass/1systems/mors.nix b/lass/1systems/mors.nix index 4ba9df6f9..9b2200c58 100644 --- a/lass/1systems/mors.nix +++ b/lass/1systems/mors.nix @@ -38,6 +38,10 @@ ../3modules/wordpress_nginx.nix ]; lass.wordpress."testserver.de" = { + multiSite = { + "1" = "testserver.de"; + "2" = "bla.testserver.de"; + }; }; services.mysql = { @@ -52,6 +56,27 @@ { predicate = "-i retiolum -p tcp --dport 80"; target = "ACCEPT"; precedence = 9998; } ]; } + { + #owncloud-test + #imports = singleton (sitesGenerators.createWordpress "testserver.de"); + imports = [ + ../3modules/owncloud_nginx.nix + ]; + lass.owncloud."owncloud-test.de" = { + }; + + #services.mysql = { + # enable = true; + # package = pkgs.mariadb; + # rootPassword = "/mysql_rootPassword"; + #}; + networking.extraHosts = '' + 10.243.0.2 owncloud-test.de + ''; + krebs.iptables.tables.filter.INPUT.rules = [ + { predicate = "-i retiolum -p tcp --dport 80"; target = "ACCEPT"; precedence = 9998; } + ]; + } ]; krebs.build.host = config.krebs.hosts.mors; @@ -59,11 +84,12 @@ networking.wireless.enable = true; networking.extraHosts = '' - 10.243.206.102 habsys.de - 10.243.206.102 pixelpocket.de - 10.243.206.102 karlaskop.de - 10.243.206.102 ubikmedia.de - 10.243.206.102 apanowicz.de + 213.239.205.240 wohnprojekt-rhh.de + 213.239.205.240 karlaskop.de + 213.239.205.240 makeup.apanowicz.de + 213.239.205.240 pixelpocket.de + 213.239.205.240 reich-gebaeudereinigung.de + 213.239.205.240 o.ubikmedia.de ''; hardware.enableAllFirmware = true; -- cgit v1.2.3 From cef2be532b0cc76071b0b3515fc71214b37591f0 Mon Sep 17 00:00:00 2001 From: makefu Date: Sat, 26 Dec 2015 10:44:56 +0100 Subject: m 3 Reaktor: add workdir/state_dir --- krebs/3modules/Reaktor.nix | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/krebs/3modules/Reaktor.nix b/krebs/3modules/Reaktor.nix index 1ec49b81e..d219d1800 100644 --- a/krebs/3modules/Reaktor.nix +++ b/krebs/3modules/Reaktor.nix @@ -62,6 +62,14 @@ let configuration appended to the default or overridden configuration ''; }; + + workdir = mkOption { + default = "/var/lib/Reaktor"; + type = types.str; + description = '' + Reaktor working directory + ''; + }; extraEnviron = mkOption { default = {}; type = types.attrsOf types.str; @@ -91,7 +99,7 @@ let # uid = config.ids.uids.Reaktor; uid = 2066439104; #genid Reaktor description = "Reaktor user"; - home = "/var/lib/Reaktor"; + home = cfg.workdir; createHome = true; }; @@ -113,6 +121,7 @@ let GIT_SSL_CAINFO = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; REAKTOR_NICKNAME = cfg.nickname; REAKTOR_DEBUG = (if cfg.debug then "True" else "False"); + state_dir = cfg.workdir; } // cfg.extraEnviron; serviceConfig= { ExecStartPre = pkgs.writeScript "Reaktor-init" '' -- cgit v1.2.3 From 8f98ae9842963d801945c850e9da1e450e098ce3 Mon Sep 17 00:00:00 2001 From: makefu Date: Sat, 26 Dec 2015 10:54:02 +0100 Subject: m 3 buildbot/master: use genid --- krebs/3modules/buildbot/master.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index 7078000fe..5870c3145 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -308,7 +308,7 @@ let imp = { users.extraUsers.buildbotMaster = { - uid = 672626386; #genid buildbotMaster + uid = genid "buildbotMaster"; description = "Buildbot Master"; home = cfg.workDir; createHome = false; -- cgit v1.2.3 From 669e4be273ac2abe9505ca6411d5ee37f1771d4c Mon Sep 17 00:00:00 2001 From: makefu Date: Sat, 26 Dec 2015 11:06:11 +0100 Subject: k 5 Reaktor/plugins: converted plugins from makefu/2/Reaktor --- krebs/5pkgs/Reaktor/plugins.nix | 124 ++++++++++++++++++++++----- krebs/5pkgs/Reaktor/scripts/random-issue.sh | 20 +++++ krebs/5pkgs/Reaktor/scripts/sed-plugin.py | 53 ++++++++++++ krebs/5pkgs/Reaktor/scripts/shack-correct.sh | 6 ++ makefu/2configs/Reaktor/full.nix | 18 ---- makefu/2configs/Reaktor/random-emoji.nix | 26 ------ makefu/2configs/Reaktor/random-emoji.sh | 6 -- makefu/2configs/Reaktor/random-issue.sh | 20 ----- makefu/2configs/Reaktor/sed-plugin.nix | 18 ---- makefu/2configs/Reaktor/sed-plugin.py | 53 ------------ makefu/2configs/Reaktor/shack-correct.nix | 20 ----- makefu/2configs/Reaktor/shack-correct.sh | 6 -- makefu/2configs/Reaktor/simpleExtend.nix | 19 ---- makefu/2configs/Reaktor/stockholmLentil.nix | 27 ------ makefu/2configs/Reaktor/titlebot.nix | 38 -------- 15 files changed, 181 insertions(+), 273 deletions(-) create mode 100644 krebs/5pkgs/Reaktor/scripts/random-issue.sh create mode 100644 krebs/5pkgs/Reaktor/scripts/sed-plugin.py create mode 100644 krebs/5pkgs/Reaktor/scripts/shack-correct.sh delete mode 100644 makefu/2configs/Reaktor/full.nix delete mode 100644 makefu/2configs/Reaktor/random-emoji.nix delete mode 100644 makefu/2configs/Reaktor/random-emoji.sh delete mode 100644 makefu/2configs/Reaktor/random-issue.sh delete mode 100644 makefu/2configs/Reaktor/sed-plugin.nix delete mode 100644 makefu/2configs/Reaktor/sed-plugin.py delete mode 100644 makefu/2configs/Reaktor/shack-correct.nix delete mode 100644 makefu/2configs/Reaktor/shack-correct.sh delete mode 100644 makefu/2configs/Reaktor/simpleExtend.nix delete mode 100644 makefu/2configs/Reaktor/stockholmLentil.nix delete mode 100644 makefu/2configs/Reaktor/titlebot.nix diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix index 05ede38e1..3b2508862 100644 --- a/krebs/5pkgs/Reaktor/plugins.nix +++ b/krebs/5pkgs/Reaktor/plugins.nix @@ -1,38 +1,118 @@ { stdenv, lib, pkgs, makeWrapper }: rec { - buildReaktorPlugin = { name - # TODO: profiles - , extraConfig + # Begin API + buildBaseReaktorPlugin = { name + , config # python extra configuration for plugin , phases ? [] , ... } @ attrs: stdenv.mkDerivation (attrs // { name = "Reaktor-plugin-" + name; - phases = phases ++ [ "installPhase" ]; isReaktorPlugin = true; }); - random-emoji = buildReaktorPlugin rec { - name = "random-emoji"; - src = ./scripts/random-emoji.sh; + buildSimpleReaktorPlugin = name: { script + , path ? [] + , env ? {} + , pattern ? "" + , ... } @ attrs: + let + path_env = { "PATH" = lib.makeSearchPath "bin" (path ++ [ pkgs.coreutils ]); }; + src_dir = pkgs.substituteAll ( { + inherit name; + dir = "bin"; + isExecutable = true; + src = script; + }); + src_file = "${src_dir}/bin/${name}"; + config = '' + public_commands.insert(0,{ + 'capname' : "${name}", + 'pattern' : ${if pattern == "" then + ''indirect_pattern.format("${name}")'' else + ''"${pattern}"'' }, + 'argv' : ["${src_file}"], + 'env' : ${builtins.toJSON path_env // env})}) + ''; + config_file = pkgs.writeText "plugin.py" config; + in buildBaseReaktorPlugin (attrs // rec { + inherit name config; + + phases = [ "installPhase" ]; + buildInputs = [ makeWrapper ]; + installPhase = '' + mkdir -p $out/bin $out/etc/Reaktor + ln -s ${src_file} $out/bin + wrapProgram $out/bin/${name} \ + --prefix PATH : ${path_env.PATH} + ln -s ${config_file} $out/etc/Reaktor/plugin.py + ''; + + }); + # End API + + # Begin Plugins + random-emoji = buildSimpleReaktorPlugin "emoji" { + path = with pkgs; [ gnused gnugrep xmlstarlet curl ]; + script = ./scripts/random-emoji.sh; + }; + + sed-plugin = buildSimpleReaktorPlugin "sed-plugin" { + path = [ pkgs.gnused ]; + # only support s///gi the plugin needs to see every msg + # TODO: this will eat up the last regex, fix Reaktor to support fallthru + pattern = "^(?P.*)$$"; + script = ./scripts/sed-plugin.py; + }; + + shack-correct = buildSimpleReaktorPlugin "shack-correct" { + path = [ pkgs.gnused ]; + pattern = "^(?P.*Shack.*)$$"; + script = ./scripts/shack-correct.sh; + }; + + nixos-version = buildSimpleReaktorPlugin "nixos-version" { + script = pkgs.writeScript "nixos-version" '' + #! /bin/sh + . /etc/os-release + echo "$PRETTY_NAME" + ''; + }; + stockholm-issue = buildSimpleReaktorPlugin "stockholm-issue" { + script = ./scripts/random-issue.sh; + path = with pkgs; [ git gnused lentil ]; + env = { "origin"= "http://cgit.gum/stockholm"; }; + }; + + titlebot = + let + pypkgs = pkgs.python3Packages; + titlebot_cmds = pypkgs.buildPythonPackage { + name = "titlebot_cmds"; + propagatedBuildInputs = with pypkgs; [ setuptools ]; + src = pkgs.fetchurl { + url = "https://github.com/makefu/reaktor-titlebot/archive/2.1.0.tar.gz"; + sha256 = "0wvf09wmk8b52f9j65qrw81nwrhs9pfhijwrlkzp5l7l2q8cjkp6"; + }; + }; + in buildBaseReaktorPlugin rec { + name = "titlebot"; phases = [ "installPhase" ]; - buildInputs = [ makeWrapper ]; installPhase = '' - mkdir -p $out/bin - install -vm 755 ${src} $out/bin/random-emoji.sh - wrapProgram $out/bin/random-emoji.sh \ - --prefix PATH : ${lib.makeSearchPath "bin" (with pkgs; [ - coreutils - gnused - gnugrep - xmlstarlet - curl])}; + mkdir -p $out + ln -s ${titlebot_cmds}/* $out ''; - extraConfig = '' - public_commands.insert(0,{ - 'capname' : "emoji", - 'pattern' : indirect_pattern.format("emoji"), - 'argv' : ["random-emoji.sh"]) + config = '' + def titlebot_cmd(cmd): + from os import environ + return { 'capname': cmd, + 'env': { 'TITLEDB': + environ['state_dir']+'/suggestions.json' }, + 'pattern': '^\\.' + cmd + '\\s*(?:\\s+(?P.*))?$$', + 'argv': [ '${titlebot_cmds}/bin/' + cmd ] } + for i in ['up','help','list','top','new']: + public_commands.insert(0,titlebot_cmd(i)) + commands.insert(0,titlebot_cmd('clear')) ''; }; } diff --git a/krebs/5pkgs/Reaktor/scripts/random-issue.sh b/krebs/5pkgs/Reaktor/scripts/random-issue.sh new file mode 100644 index 000000000..5c47c6156 --- /dev/null +++ b/krebs/5pkgs/Reaktor/scripts/random-issue.sh @@ -0,0 +1,20 @@ +#! /bin/sh +set -eu +# requires env: +# $state_dir +# $origin + +# in PATH: git,lentil,coreutils +subdir=`echo "$1" | tr -dc "[:alnum:]"` +name=`echo "$origin" | tr -dc "[:alnum:]"` +track="$state_dir/$name-checkout" +(if test -e "$track" ;then + cd "$track" + git fetch origin master + git reset --hard origin/master +else + git clone "$origin" "$track" +fi) >&2 + +cd "$track" +lentil "${subdir:-.}" -f csv | sed 1d | shuf | head -1 diff --git a/krebs/5pkgs/Reaktor/scripts/sed-plugin.py b/krebs/5pkgs/Reaktor/scripts/sed-plugin.py new file mode 100644 index 000000000..8103c9585 --- /dev/null +++ b/krebs/5pkgs/Reaktor/scripts/sed-plugin.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +# Usage: +# _from=krebs state_dir=. python sed-plugin.py 'dick butt' +# _from=krebs state_dir=. python sed-plugin.py 's/t/l/g' +## dick bull +import shelve +from os import environ +from os.path import join +from sys import argv +d = shelve.open(join(environ['state_dir'],'sed-plugin.shelve'),writeback=True) +usr = environ['_from'] +import re + +def is_regex(line): + myre = re.compile(r'^s/((?:\\/|[^/])+)/((?:\\/|[^/])*)/([ig]*)$') + return myre.match(line) + +line = argv[1] +m = is_regex(line) + +if m: + f,t,flagstr = m.groups() + fn = f.replace('\/','/') + tn = t.replace('\/','/') + flags = 0 + count = 1 + if flagstr: + if 'i' in flagstr: + flags = re.IGNORECASE + if 'g' in flagstr: + count = 0 + else: + flagstr = '' + last = d.get(usr,None) + if last: + #print(re.sub(fn,tn,last,count=count,flags=flags)) + from subprocess import Popen,PIPE + p = Popen(['sed','s/{}/{}/{}'.format(f,t,flagstr)],stdin=PIPE,stdout=PIPE ) + so,se = p.communicate(bytes("{}\n".format(last),"UTF-8")) + if p.returncode: + print("something went wrong when trying to process your regex: {}".format(se.decode())) + ret = so.decode() + print("\x1b[1m{}\x1b[0m meinte: {}".format(usr,ret.strip())) + if ret: + d[usr] = ret + + else: + print("no last message") +else: + d[usr] = line + +d.close() diff --git a/krebs/5pkgs/Reaktor/scripts/shack-correct.sh b/krebs/5pkgs/Reaktor/scripts/shack-correct.sh new file mode 100644 index 000000000..3b4d04f80 --- /dev/null +++ b/krebs/5pkgs/Reaktor/scripts/shack-correct.sh @@ -0,0 +1,6 @@ +#! /bin/sh +set -eu +printf "Sie meinten wohl \"" +echo -n $@ | sed 's/Shack/shack/g' +echo "\"" +echo "${_from}--" diff --git a/makefu/2configs/Reaktor/full.nix b/makefu/2configs/Reaktor/full.nix deleted file mode 100644 index 50620890f..000000000 --- a/makefu/2configs/Reaktor/full.nix +++ /dev/null @@ -1,18 +0,0 @@ -_: -{ - # implementation of the complete Reaktor bot - imports = [ - #./stockholmLentil.nix - ./simpleExtend.nix - ./random-emoji.nix - ./titlebot.nix - ./shack-correct.nix - ./sed-plugin.nix - ]; - krebs.Reaktor.nickname = "Reaktor|bot"; - krebs.Reaktor.enable = true; - - krebs.Reaktor.extraEnviron = { - REAKTOR_CHANNELS = "#krebs,#binaergewitter,#shackspace"; - }; -} diff --git a/makefu/2configs/Reaktor/random-emoji.nix b/makefu/2configs/Reaktor/random-emoji.nix deleted file mode 100644 index 3113a826b..000000000 --- a/makefu/2configs/Reaktor/random-emoji.nix +++ /dev/null @@ -1,26 +0,0 @@ -{ config, lib, pkgs, ... }: - -with pkgs; -let - rpkg = pkgs.substituteAll( { - name="random-emoji"; - dir= "bin"; - isExecutable=true; - src= ./random-emoji.sh; - }); - rpkg-path = lib.makeSearchPath "bin" (with pkgs; [ - coreutils - gnused - gnugrep - xmlstarlet - curl]); -in { - # TODO: make origin a variable, <- module is generic enough to handle different origins, not only stockholm - krebs.Reaktor.extraConfig = '' - public_commands.insert(0,{ - 'capname' : "emoji", - 'pattern' : indirect_pattern.format("emoji"), - 'argv' : ["${rpkg}/bin/random-emoji"], - 'env' : { 'PATH':'${rpkg-path}' } }) - ''; -} diff --git a/makefu/2configs/Reaktor/random-emoji.sh b/makefu/2configs/Reaktor/random-emoji.sh deleted file mode 100644 index 386aa68b9..000000000 --- a/makefu/2configs/Reaktor/random-emoji.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -curl http://emojicons.com/random -s | \ - grep data-text | \ - sed -n 's/.*>\(.*\)<\/textarea>/\1/p' | \ - head -n 1 | \ - xmlstarlet unesc diff --git a/makefu/2configs/Reaktor/random-issue.sh b/makefu/2configs/Reaktor/random-issue.sh deleted file mode 100644 index 5c47c6156..000000000 --- a/makefu/2configs/Reaktor/random-issue.sh +++ /dev/null @@ -1,20 +0,0 @@ -#! /bin/sh -set -eu -# requires env: -# $state_dir -# $origin - -# in PATH: git,lentil,coreutils -subdir=`echo "$1" | tr -dc "[:alnum:]"` -name=`echo "$origin" | tr -dc "[:alnum:]"` -track="$state_dir/$name-checkout" -(if test -e "$track" ;then - cd "$track" - git fetch origin master - git reset --hard origin/master -else - git clone "$origin" "$track" -fi) >&2 - -cd "$track" -lentil "${subdir:-.}" -f csv | sed 1d | shuf | head -1 diff --git a/makefu/2configs/Reaktor/sed-plugin.nix b/makefu/2configs/Reaktor/sed-plugin.nix deleted file mode 100644 index a451e0d3e..000000000 --- a/makefu/2configs/Reaktor/sed-plugin.nix +++ /dev/null @@ -1,18 +0,0 @@ -{ config, lib, pkgs, ... }: - -with pkgs; -let - script = ./sed-plugin.py; -in { - #TODO: this will eat up the last regex, fix Reaktor - krebs.Reaktor.extraConfig = '' - public_commands.append({ - 'capname' : "sed-plugin", - # only support s///gi - 'pattern' : '^(?P.*)$$', - 'argv' : ["${pkgs.python3}/bin/python3","${script}"], - 'env' : { 'state_dir' : workdir, - 'PATH':'${lib.makeSearchPath "bin" [pkgs.gnused]}' }}) - ''; -} - diff --git a/makefu/2configs/Reaktor/sed-plugin.py b/makefu/2configs/Reaktor/sed-plugin.py deleted file mode 100644 index 8103c9585..000000000 --- a/makefu/2configs/Reaktor/sed-plugin.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 - -# Usage: -# _from=krebs state_dir=. python sed-plugin.py 'dick butt' -# _from=krebs state_dir=. python sed-plugin.py 's/t/l/g' -## dick bull -import shelve -from os import environ -from os.path import join -from sys import argv -d = shelve.open(join(environ['state_dir'],'sed-plugin.shelve'),writeback=True) -usr = environ['_from'] -import re - -def is_regex(line): - myre = re.compile(r'^s/((?:\\/|[^/])+)/((?:\\/|[^/])*)/([ig]*)$') - return myre.match(line) - -line = argv[1] -m = is_regex(line) - -if m: - f,t,flagstr = m.groups() - fn = f.replace('\/','/') - tn = t.replace('\/','/') - flags = 0 - count = 1 - if flagstr: - if 'i' in flagstr: - flags = re.IGNORECASE - if 'g' in flagstr: - count = 0 - else: - flagstr = '' - last = d.get(usr,None) - if last: - #print(re.sub(fn,tn,last,count=count,flags=flags)) - from subprocess import Popen,PIPE - p = Popen(['sed','s/{}/{}/{}'.format(f,t,flagstr)],stdin=PIPE,stdout=PIPE ) - so,se = p.communicate(bytes("{}\n".format(last),"UTF-8")) - if p.returncode: - print("something went wrong when trying to process your regex: {}".format(se.decode())) - ret = so.decode() - print("\x1b[1m{}\x1b[0m meinte: {}".format(usr,ret.strip())) - if ret: - d[usr] = ret - - else: - print("no last message") -else: - d[usr] = line - -d.close() diff --git a/makefu/2configs/Reaktor/shack-correct.nix b/makefu/2configs/Reaktor/shack-correct.nix deleted file mode 100644 index 8f30807f1..000000000 --- a/makefu/2configs/Reaktor/shack-correct.nix +++ /dev/null @@ -1,20 +0,0 @@ -{ config, lib, pkgs, ... }: - -with pkgs; -let - script = pkgs.substituteAll ( { - name="shack-correct"; - isExecutable=true; - dir = ""; - src = ./shack-correct.sh; - }); -in { - krebs.Reaktor.extraConfig = '' - public_commands.insert(0,{ - 'capname' : "shack-correct", - 'pattern' : '^(?P.*Shack.*)$$', - 'argv' : ["${script}"], - 'env' : { }}) - ''; -} - diff --git a/makefu/2configs/Reaktor/shack-correct.sh b/makefu/2configs/Reaktor/shack-correct.sh deleted file mode 100644 index 3b4d04f80..000000000 --- a/makefu/2configs/Reaktor/shack-correct.sh +++ /dev/null @@ -1,6 +0,0 @@ -#! /bin/sh -set -eu -printf "Sie meinten wohl \"" -echo -n $@ | sed 's/Shack/shack/g' -echo "\"" -echo "${_from}--" diff --git a/makefu/2configs/Reaktor/simpleExtend.nix b/makefu/2configs/Reaktor/simpleExtend.nix deleted file mode 100644 index 95175a4e0..000000000 --- a/makefu/2configs/Reaktor/simpleExtend.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ config, lib, pkgs, ... }: - -with pkgs; -let - nixos-version-script = pkgs.writeScript "nix-version" '' - #! /bin/sh - . /etc/os-release - echo "$PRETTY_NAME" - ''; -in { - krebs.Reaktor.extraConfig = '' - public_commands.insert(0,{ - 'capname' : "nixos-version", - 'pattern' : indirect_pattern.format("nixos-version"), - 'argv' : ["${nixos-version-script}"], - 'env' : { 'state_dir': workdir } }) - ''; -} - diff --git a/makefu/2configs/Reaktor/stockholmLentil.nix b/makefu/2configs/Reaktor/stockholmLentil.nix deleted file mode 100644 index 21f0305fb..000000000 --- a/makefu/2configs/Reaktor/stockholmLentil.nix +++ /dev/null @@ -1,27 +0,0 @@ -{ config, lib, pkgs, ... }: - -with pkgs; -let - random-issue = pkgs.substituteAll( { - name="random-issue"; - dir= "bin"; - isExecutable=true; - src= ./random-issue.sh; - }); - random-issue-path = lib.makeSearchPath "bin" (with pkgs; [ - coreutils - git - gnused - lentil]); -in { - # TODO: make origin a variable, <- module is generic enough to handle different origins, not only stockholm - krebs.Reaktor.extraConfig = '' - public_commands.insert(0,{ - 'capname' : "stockholm-issue", - 'pattern' : indirect_pattern.format("stockholm-issue"), - 'argv' : ["${random-issue}/bin/random-issue"], - 'env' : { 'state_dir': workdir, - 'PATH':'${random-issue-path}', - 'origin':'http://cgit.pnp/stockholm' } }) - ''; -} diff --git a/makefu/2configs/Reaktor/titlebot.nix b/makefu/2configs/Reaktor/titlebot.nix deleted file mode 100644 index 9ef02548b..000000000 --- a/makefu/2configs/Reaktor/titlebot.nix +++ /dev/null @@ -1,38 +0,0 @@ -{ stdenv,config, lib, pkgs, ... }: - -with pkgs; -let - pypkgs = pkgs.python3Packages; - titlebot_cmds = pypkgs.buildPythonPackage { - name = "titlebot_cmds"; - propagatedBuildInputs = with pypkgs; [ setuptools ]; - src = fetchurl { - # https://github.com/makefu/reaktor-titlebot tag 2.1.0 - url = "https://github.com/makefu/reaktor-titlebot/archive/2.1.0.tar.gz"; - sha256 = "0wvf09wmk8b52f9j65qrw81nwrhs9pfhijwrlkzp5l7l2q8cjkp6"; - }; - }; - pub_cmds = ["up" "help" "list" "top" "highest" "undo" ]; - priv_cmds = [ "clear" ]; -in { - # TODO: write identify file in - # {config.users.extraUsers.Reaktor.home}/state/admin.lst - krebs.Reaktor.extraConfig = '' - def titlebot_cmd(cmd): - return { - 'capname': cmd, - 'env': { - 'TITLEDB': - '${config.users.extraUsers.Reaktor.home}/suggestions.json' - }, - 'pattern': '^\\.' + cmd + '\\s*(?:\\s+(?P.*))?$$', - 'argv': [ '${titlebot_cmds}/bin/' + cmd ] } - # TODO: for each element in ${titlebot_cmds}/bin/* - public_commands.insert(0,titlebot_cmd('up')) - public_commands.insert(0,titlebot_cmd('help')) - public_commands.insert(0,titlebot_cmd('list')) - public_commands.insert(0,titlebot_cmd('top')) - public_commands.insert(0,titlebot_cmd('new')) - commands.insert(0,titlebot_cmd('clear')) - ''; -} -- cgit v1.2.3 From a2f5e7e320bb0fbca0a0694d91e4fb20dc4ef329 Mon Sep 17 00:00:00 2001 From: makefu Date: Sat, 26 Dec 2015 11:31:09 +0100 Subject: k 5 ReaktorPlugins: hotfix for env generation --- krebs/5pkgs/Reaktor/plugins.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix index 3b2508862..b1a61d3fa 100644 --- a/krebs/5pkgs/Reaktor/plugins.nix +++ b/krebs/5pkgs/Reaktor/plugins.nix @@ -32,7 +32,7 @@ rec { ''indirect_pattern.format("${name}")'' else ''"${pattern}"'' }, 'argv' : ["${src_file}"], - 'env' : ${builtins.toJSON path_env // env})}) + 'env' : ${builtins.toJSON (path_env // env)})}) ''; config_file = pkgs.writeText "plugin.py" config; in buildBaseReaktorPlugin (attrs // rec { -- cgit v1.2.3 From 7bed1761bdbfc3fc7e2df56dcf069511eec2a97d Mon Sep 17 00:00:00 2001 From: makefu Date: Sat, 26 Dec 2015 11:41:41 +0100 Subject: m 3 Reaktor: now supports plugin infra see m/1/pornocauster --- krebs/3modules/Reaktor.nix | 5 ++++- krebs/5pkgs/Reaktor/plugins.nix | 4 ++-- makefu/1systems/pornocauster.nix | 13 +++++++------ 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/krebs/3modules/Reaktor.nix b/krebs/3modules/Reaktor.nix index 59058bffc..607eb7cac 100644 --- a/krebs/3modules/Reaktor.nix +++ b/krebs/3modules/Reaktor.nix @@ -9,6 +9,7 @@ let ${cfg.overrideConfig} '' else ""} ## Extra Config + ${concatStringsSep "\n" (map (plug: plug.config) cfg.plugins)} ${cfg.extraConfig} ''; cfg = config.krebs.Reaktor; @@ -35,7 +36,6 @@ let ''; }; - overrideConfig = mkOption { default = null; type = types.nullOr types.str; @@ -44,6 +44,9 @@ let Reaktor default cfg can be retrieved via `reaktor get-config` ''; }; + plugins = mkOption { + default = [pkgs.ReaktorPlugins.nixos-version]; + }; extraConfig = mkOption { default = ""; type = types.string; diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix index b1a61d3fa..5c7b89f5c 100644 --- a/krebs/5pkgs/Reaktor/plugins.nix +++ b/krebs/5pkgs/Reaktor/plugins.nix @@ -32,7 +32,7 @@ rec { ''indirect_pattern.format("${name}")'' else ''"${pattern}"'' }, 'argv' : ["${src_file}"], - 'env' : ${builtins.toJSON (path_env // env)})}) + 'env' : ${builtins.toJSON (path_env // env)} }) ''; config_file = pkgs.writeText "plugin.py" config; in buildBaseReaktorPlugin (attrs // rec { @@ -81,7 +81,7 @@ rec { stockholm-issue = buildSimpleReaktorPlugin "stockholm-issue" { script = ./scripts/random-issue.sh; path = with pkgs; [ git gnused lentil ]; - env = { "origin"= "http://cgit.gum/stockholm"; }; + env = { "origin" = "http://cgit.gum/stockholm"; }; }; titlebot = diff --git a/makefu/1systems/pornocauster.nix b/makefu/1systems/pornocauster.nix index 28b77d330..690e26b36 100644 --- a/makefu/1systems/pornocauster.nix +++ b/makefu/1systems/pornocauster.nix @@ -26,6 +26,7 @@ # services ../2configs/git/brain-retiolum.nix ../2configs/tor.nix + # ../2configs/buildbot-standalone.nix # hardware specifics are in here ../2configs/hw/tp-x220.nix @@ -36,14 +37,14 @@ ]; nixpkgs.config.packageOverrides = pkgs: { tinc = pkgs.tinc_pre; - buildbot = let - pkgs1509 = import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz) {}; - in pkgs1509.buildbot; }; - makefu.buildbot.master.enable = true; - #krebs.Reaktor.enable = true; - #krebs.Reaktor.nickname = "makefu|r"; + krebs.Reaktor = { + enable = true; + nickname = "makefu|r"; + plugins = with pkgs.ReaktorPlugins; [ nixos-version random-emoji ]; + }; + # nix.binaryCaches = [ "http://acng.shack/nixos" "https://cache.nixos.org" ]; environment.systemPackages = with pkgs;[ -- cgit v1.2.3 From 70d0cae1d3831cd24b3e2ac68a927132f71d2801 Mon Sep 17 00:00:00 2001 From: lassulus Date: Sat, 26 Dec 2015 17:30:59 +0100 Subject: l 2: move pkgs from xserver to baseX --- lass/2configs/baseX.nix | 9 +++++++-- lass/2configs/xserver/default.nix | 14 -------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/lass/2configs/baseX.nix b/lass/2configs/baseX.nix index 4e46c18d2..0596682df 100644 --- a/lass/2configs/baseX.nix +++ b/lass/2configs/baseX.nix @@ -31,10 +31,15 @@ in { environment.systemPackages = with pkgs; [ - powertop - sxiv + gitAndTools.qgit + mpv much + pavucontrol + powertop push + slock + sxiv + xsel zathura #window manager stuff diff --git a/lass/2configs/xserver/default.nix b/lass/2configs/xserver/default.nix index da337f6a7..04d14c7ce 100644 --- a/lass/2configs/xserver/default.nix +++ b/lass/2configs/xserver/default.nix @@ -7,7 +7,6 @@ let user = config.users.users.mainUser; out = { - services.xserver = { display = 11; tty = 11; @@ -41,16 +40,6 @@ let }; }; - environment.systemPackages = [ - pkgs.gitAndTools.qgit - pkgs.mpv - pkgs.pavucontrol - pkgs.slock - pkgs.sxiv - pkgs.xsel - pkgs.zathura - ]; - security.setuidPrograms = [ "slock" ]; @@ -106,9 +95,6 @@ let set -efu export PATH; PATH=${makeSearchPath "bin" ([ pkgs.rxvt_unicode - pkgs.i3lock - pkgs.haskellPackages.yeganesh - pkgs.dmenu ] ++ config.environment.systemPackages)}:/var/setuid-wrappers settle() {( # Use PATH for a clean journal -- cgit v1.2.3 From f22fe4e5d97237dbe76bc856909950487634c7be Mon Sep 17 00:00:00 2001 From: lassulus Date: Sat, 26 Dec 2015 17:31:38 +0100 Subject: l 5 newsbot-js: rev 6ee4884 -> 802b172 --- lass/5pkgs/newsbot-js/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lass/5pkgs/newsbot-js/default.nix b/lass/5pkgs/newsbot-js/default.nix index 0d194e6f3..cabd7422c 100644 --- a/lass/5pkgs/newsbot-js/default.nix +++ b/lass/5pkgs/newsbot-js/default.nix @@ -26,8 +26,8 @@ in nodePackages.buildNodePackage { src = fetchgit { url = "http://cgit.echelon/newsbot-js/"; - rev = "6ee488430c6915eeae03f1569084577d39cef51d"; - sha256 = "00xmn7hzcs0mm6hjf5i37d9nna5rcd0gra0ynch7x2id8liazksx"; + rev = "802b172d0eed6c9625a9cb5db408f5cc8c01784e"; + sha256 = "794fc7845aca311f7cf7b6bdc109b5a25d0e2299322bc6612edadc477b2536e2"; }; phases = [ -- cgit v1.2.3 From 743842268327b5fd12ba4d19b6260d47535976a3 Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 28 Dec 2015 10:58:13 +0100 Subject: k 5 default: populate supports infesting arg by setting infesting for populate, data will be written to /mnt instead of root. --- krebs/default.nix | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/krebs/default.nix b/krebs/default.nix index ad0205426..81ddd3ea6 100644 --- a/krebs/default.nix +++ b/krebs/default.nix @@ -36,6 +36,7 @@ let out = { { system ? current-host-name , target ? system }@args: let + config = get-config system; in '' #! /bin/sh # ${current-date} ${current-user-name}@${current-host-name} @@ -47,6 +48,10 @@ let out = { ${builtins.readFile ./4lib/infest/install-nix.sh} ''} + # Prepare target source via bind-mounting + + (${populate (args // { infesting = true;}) }) + (${nixos-install args}) ${rootssh target '' @@ -98,7 +103,6 @@ let out = { #! /bin/sh # ${current-date} ${current-user-name}@${current-host-name} # krebs.nixos-install - (${populate args}) ${rootssh target '' export PATH; PATH=/root/.nix-profile/bin:$PATH @@ -205,6 +209,7 @@ let out = { populate = { system ? current-host-name , target ? system + , infesting ? false }@args: let out = '' #! /bin/sh @@ -217,6 +222,8 @@ let out = { ["dir" "git"])} ''; + + target_prefix=lib.optionalString infesting "/mnt"; config = get-config system; current-host = config.krebs.hosts.${current-host-name}; @@ -225,17 +232,18 @@ let out = { methods.dir = config: let can-push = config.host.name == current-host.name; + target-path = target_prefix + config.target-path; push-method = '' rsync \ --exclude .git \ --exclude .graveyard \ --exclude old \ --exclude tmp \ - --rsync-path='mkdir -p ${config.target-path} && rsync' \ + --rsync-path='mkdir -p ${target-path} && rsync' \ --delete-excluded \ -vrLptgoD \ ${config.path}/ \ - root@${target}:${config.target-path} + root@${target}:${target-path} ''; in if can-push then push-method else @@ -244,9 +252,10 @@ let out = { throw "No way to push ${dir} from ${current-host.name} to ${target}"; methods.git = config: - rootssh target '' - mkdir -p ${config.target-path} - cd ${config.target-path} + let target-path = target_prefix + config.target-path; + in rootssh target '' + mkdir -p ${target-path} + cd ${target-path} if ! test -e .git; then git init fi -- cgit v1.2.3 From 95caa8d7fb6f72cbf5064256f71750096d32a6c0 Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 28 Dec 2015 13:23:34 +0100 Subject: k 5 test/infest-cac-centos7: use defer, loop until we get a working cac box --- krebs/5pkgs/test/infest-cac-centos7/notes | 77 ++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 26 deletions(-) diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes index cfb074423..3f4fcd859 100755 --- a/krebs/5pkgs/test/infest-cac-centos7/notes +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -8,6 +8,17 @@ set -eufx krebs_cred=${krebs_cred-./cac.json} retiolum_key=${retiolum_key-./retiolum.rsa_key.priv} +clear_defer(){ + echo "${trapstr:-exit}" + trap - INT TERM EXIT KILL +} +defer(){ + if test -z "${debug:-}"; then + trapstr="$1;${trapstr:-exit}" + trap "$trapstr" INT TERM EXIT KILL + fi +} + # Sanity if test ! -r "$krebs_cred";then echo "\$krebs_cred=$krebs_cred must be readable"; exit 1 @@ -24,8 +35,8 @@ export cac_servers_cache=$krebs_secrets/servers_cache.json export cac_tasks_cache=$krebs_secrets/tasks_cache.json export cac_templates_cache=$krebs_secrets/templates_cache.json # we need to receive this key from buildmaster to speed up tinc bootstrap -TRAP="rm -r $krebs_secrets;trap - INT TERM EXIT" -trap "$TRAP" INT TERM EXIT +defer "trap - INT TERM EXIT" +defer "rm -r $krebs_secrets" cat > $sec_file <&1\ - | jq -r .servername) - -id=servername:$name -trap "cac delete $id;$TRAP;exit" INT TERM EXIT -# TODO: timeout? - -wait_login_cac(){ - # timeout - for t in `seq 180`;do - # now we have a working cac server - if cac ssh $1 -o ConnectTimeout=10 \ - cat /etc/redhat-release | \ - grep CentOS ;then - return 0 - fi - sleep 10 - done - return 1 -} -# die on timeout -wait_login_cac $id +# preserve old trap +old_trapstr=$(clear_defer) +while true;do + # Template 26: CentOS7 + # TODO: use cac templates to determine the real Centos7 template in case it changes + name=$( cac build cpu=1 ram=512 storage=10 os=26 2>&1\ + | jq -r .servername) + id=servername:$name + + clear_defer >/dev/null + defer "cac delete $id" + + # TODO: timeout? + + wait_login_cac(){ + # we wait for 15 minutes + for t in `seq 90`;do + # now we have a working cac server + if cac ssh $1 -o ConnectTimeout=10 \ + cat /etc/redhat-release | \ + grep CentOS ;then + return 0 + fi + sleep 10 + done + return 1 + } + # die on timeout + if ! wait_login_cac $id;then + echo "unable to boot a working system within time frame, retrying..." >&2 + echo "Cleaning up old image,last status: $(cac update;cac getserver $id | jq -r .status)" + eval "$(clear_defer)" + else + echo "got a working system" >&2 + fi +done +clear_defer >/dev/null +defer "cac delete $id;$old_trapstr" mkdir -p shared/2configs/temp cac generatenetworking $id > \ -- cgit v1.2.3 From 246116dabbe849e75612fbdb57b01696913ff27e Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 28 Dec 2015 13:29:11 +0100 Subject: m 2 urlwatch: replace url for cvs2svn --- makefu/2configs/urlwatch.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefu/2configs/urlwatch.nix b/makefu/2configs/urlwatch.nix index eadffa7dd..e4f639d5b 100644 --- a/makefu/2configs/urlwatch.nix +++ b/makefu/2configs/urlwatch.nix @@ -12,7 +12,7 @@ http://git.sysphere.org/vicious/log/?qt=grep&q=Next+release https://pypi.python.org/simple/bepasty/ https://pypi.python.org/simple/xstatic/ - http://cvs2svn.tigris.org/servlets/ProjectDocumentList?folderID=2976 + http://cvs2svn.tigris.org/svn/cvs2svn/tags/ ]; }; } -- cgit v1.2.3 From 676d0f748138f0e1fa3cb2177b5a08a857f17fac Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 29 Dec 2015 12:52:35 +0100 Subject: k Zhosts: init bobby (miefda) --- krebs/Zhosts/bobby | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 krebs/Zhosts/bobby diff --git a/krebs/Zhosts/bobby b/krebs/Zhosts/bobby new file mode 100644 index 000000000..aac6e377b --- /dev/null +++ b/krebs/Zhosts/bobby @@ -0,0 +1,11 @@ +Subnet = 10.243.111.112/32 +Subnet = 42:0:0:0:0:0:111:112/128 + +-----BEGIN RSA PUBLIC KEY----- +MIIBCgKCAQEA+AScnIqFdzGl+iRZTNZ7r91n/r1H4GzDsrAupUvJ4mi7nDN4eP8s +uLvKtJp22RxfuF3Kf4KhHb8LHQ8bLLN/KDaNDXrCNBc69d7vvLsjoY+wfGLJNu4Y +Ad/8J4r3rdb83mTA3IHb47T/70MERPBr2gF84YiG6ZoQrPQuTk4lHxaI83SOhjny +0F0ucS/rBV6Vv9y5/756TKi1cFPSpY4X+qeWc8xWrBGJcJiiqYb8ZX2o/lkAJ5c+ +jI/VdybGFVGY9+bp4Jw5xBIo5KGuFnm8+blRmSDDl3joRneKQSx9FAu7RUwoajBu +cEbi1529NReQzIFT6Vt22ymbHftxOiuh4QIDAQAB +-----END RSA PUBLIC KEY----- -- cgit v1.2.3 From 5fb35e60a8e1b997473e7657d342802d533cd070 Mon Sep 17 00:00:00 2001 From: lassulus Date: Tue, 29 Dec 2015 19:30:57 +0100 Subject: k hosts: add kebsco --- krebs/Zhosts/kebsco | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 krebs/Zhosts/kebsco diff --git a/krebs/Zhosts/kebsco b/krebs/Zhosts/kebsco new file mode 100644 index 000000000..2fd1c5f42 --- /dev/null +++ b/krebs/Zhosts/kebsco @@ -0,0 +1,11 @@ +Subnet = 10.243.212.68 +Subnet = 42:9d30:3845:c822:988b:96c5:39ab:90b7 + +-----BEGIN RSA PUBLIC KEY----- +MIIBCgKCAQEA0dEwTZh2uzJpP9GL7YRyiLuezJqYiJ8/4Bl4IPshJnuO9IGbEcto +0cFm9uM9gxxqggfaCi96DsIQNlyqff2vDfEj3mdIu9T3tkRROByQF8y1NWX29NyH +zZEX8Ri8u4U2KdYTEzPXEFxBEl0GQX9mMtlvwzCq7V4ueCcWB1xDA+DtJjpd894z +3FOw0rIxYmfYhLAL5B3rzF74bcHFGV30f4JWq11wLBkyR6/Q5gxgZzkKYGwdZ/SN +C6gg86abKdp65/Wq5P331IbwPBal1ZhGbaAo1y7JpjpLvZytI2jboXeQuPZ8P5hU +L3zKKceAibPKrw9+y8lb+IKoYLF7I1KYIwIDAQAB +-----END RSA PUBLIC KEY----- -- cgit v1.2.3 From 9abd00f6af48676f08b6afdddd03d12410a1d1cd Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 29 Dec 2015 21:20:11 +0100 Subject: k 3 makefu: add ssh pubkeys to hosts --- krebs/3modules/makefu/default.nix | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/krebs/3modules/makefu/default.nix b/krebs/3modules/makefu/default.nix index 1970a0777..31516d591 100644 --- a/krebs/3modules/makefu/default.nix +++ b/krebs/3modules/makefu/default.nix @@ -83,6 +83,9 @@ with lib; ''; }; }; + ssh.privkey.path = ; + ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHDM0E608d/6rGzXqGbNSuMb2RlCojCJSiiz6QcPOC2G root@pornocauster"; + }; vbob = { @@ -108,6 +111,8 @@ with lib; ''; }; }; + ssh.privkey.path = ; + ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICPLTMl+thSq77cjYa2XF7lz5fA7JMftrLo8Dy/OBXSg root@nixos"; }; flap = rec { cores = 1; @@ -238,6 +243,8 @@ with lib; ''; }; }; + ssh.privkey.path = ; + ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH4Tjx9qK6uWtxT1HCpeC0XvDZKO/kaPygyKatpAqU6I root@wry"; }; filepimp = rec { cores = 1; @@ -287,6 +294,8 @@ with lib; ''; }; }; + ssh.privkey.path = ; + ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIujMZ3ZFxKpWeB/cjfKfYRr77+VRZk0Eik+92t03NoA root@servarch"; }; gum = rec { cores = 1; @@ -327,6 +336,8 @@ with lib; ''; }; }; + ssh.privkey.path = ; + ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIcxWFEPzke/Sdd9qNX6rSJgXal8NmINYajpFCxXfYdj root@gum"; }; }; users = addNames rec { -- cgit v1.2.3 From 4b6cd401a85cdc7aab150208cc5310645a7e59e2 Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 29 Dec 2015 21:20:36 +0100 Subject: m 1 gum: add smart monitor --- makefu/1systems/gum.nix | 3 ++- makefu/2configs/smart-monitor.nix | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 makefu/2configs/smart-monitor.nix diff --git a/makefu/1systems/gum.nix b/makefu/1systems/gum.nix index 93fb3dc3a..1907424ec 100644 --- a/makefu/1systems/gum.nix +++ b/makefu/1systems/gum.nix @@ -6,11 +6,11 @@ let internal-ip = head config.krebs.build.host.nets.retiolum.addrs4; in { imports = [ - # TODO: copy this config or move to krebs ../2configs/tinc-basic-retiolum.nix ../2configs/headless.nix ../2configs/fs/simple-swap.nix ../2configs/fs/single-partition-ext4.nix + ../2configs/smart-monitor.nix # ../2configs/iodined.nix ../2configs/git/cgit-retiolum.nix ../2configs/mattermost-docker.nix @@ -18,6 +18,7 @@ in { ../2configs/exim-retiolum.nix ../2configs/urlwatch.nix + ]; diff --git a/makefu/2configs/smart-monitor.nix b/makefu/2configs/smart-monitor.nix new file mode 100644 index 000000000..7086f622b --- /dev/null +++ b/makefu/2configs/smart-monitor.nix @@ -0,0 +1,18 @@ +{ config, ... }: +{ + services.smartd = { + enable = true; + notifications = { + mail = { + enable = true; + recipient = config.krebs.users.makefu.mail; + }; + }; + # short daily, long weekly, check on boot + defaults.monitored = "-a -o on -s (S/../.././02|L/../../7/04)"; + + devices = [{ + device = "/dev/sda"; + }]; + }; +} -- cgit v1.2.3 From 81badfd47ede4cf3860e7006c13586340415ade5 Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 29 Dec 2015 21:21:04 +0100 Subject: m 2 urlwatch: use py2 instead of py2k urlwatch --- makefu/2configs/urlwatch.nix | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/makefu/2configs/urlwatch.nix b/makefu/2configs/urlwatch.nix index e4f639d5b..a83279ba2 100644 --- a/makefu/2configs/urlwatch.nix +++ b/makefu/2configs/urlwatch.nix @@ -1,6 +1,22 @@ -{ config, ... }: +{ config, lib, ... }: { + nixpkgs.config.packageOverrides = pkgs: { + urlwatch = with pkgs.pythonPackages; buildPythonPackage rec { + name = "urlwatch-1.18"; + + propagatedBuildInputs = [ futures ]; + + src = pkgs.fetchurl { + url = "http://thp.io/2008/urlwatch/${name}.tar.gz"; + sha256 = "090qfgx249ks7103sap6w47f8302ix2k46wxhfssxwsqcqdl25vb"; + }; + + postFixup = '' + wrapProgram "$out/bin/urlwatch" --prefix "PYTHONPATH" : "$PYTHONPATH" + ''; + }; + }; krebs.urlwatch = { enable = true; mailto = config.krebs.users.makefu.mail; @@ -12,7 +28,7 @@ http://git.sysphere.org/vicious/log/?qt=grep&q=Next+release https://pypi.python.org/simple/bepasty/ https://pypi.python.org/simple/xstatic/ - http://cvs2svn.tigris.org/svn/cvs2svn/tags/ + http://guest:derpi@cvs2svn.tigris.org/svn/cvs2svn/tags/ ]; }; } -- cgit v1.2.3 From d574c0ef78f7572aec88e484d3ff6256247e878c Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 30 Dec 2015 01:38:33 +0100 Subject: m 3 buildbot/master: add secrets --- krebs/3modules/buildbot/master.nix | 16 +++++++++++++--- shared/2configs/buildbot-standalone.nix | 8 +++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index 5870c3145..74385a433 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -132,6 +132,16 @@ let ''; }; + secrets = mkOption { + default = []; + type = types.listOf types.str; + example = [ "cac.json" ]; + description = '' + List of all the secrets in which should be copied into the + buildbot master directory. + ''; + }; + slaves = mkOption { default = {}; type = types.attrsOf types.str; @@ -344,10 +354,10 @@ let fi # always override the master.cfg cp ${buildbot-master-config} ${workdir}/master.cfg + # copy secrets - cp ${secretsdir}/cac.json ${workdir} - cp ${secretsdir}/retiolum-ci.rsa_key.priv \ - ${workdir}/retiolum.rsa_key.priv + ${ concatMapStringsSep "\n" + (f: "cp ${secretsdir}/${f} ${workdir}/${f}" ) cfg.secrets } # sanity ${buildbot}/bin/buildbot checkconfig ${workdir} diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix index 51c600329..28d1eef2e 100644 --- a/shared/2configs/buildbot-standalone.nix +++ b/shared/2configs/buildbot-standalone.nix @@ -8,6 +8,9 @@ in { }; networking.firewall.allowedTCPPorts = [ 8010 9989 ]; krebs.buildbot.master = { + secrets = [ + "cac.json" + ]; slaves = { testslave = "krebspass"; omo = "krebspass"; @@ -93,9 +96,8 @@ in { # slave needs 2 files: # * cac.json # * retiolum - for file in ["cac.json", "retiolum.rsa_key.priv"]: - s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/{}".format(file), - slavedest=file)) + s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/cac.json", slavedest="cac.json")) + s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/retiolum-ci.rsa_key.priv", slavedest="retiolum.rsa_key.priv")) addShell(s, name="infest-cac-centos7",env=env, sigtermTime=60, # SIGTERM 1 minute before SIGKILL -- cgit v1.2.3 From a8cb9d41bdefe8a5dc72ca76b3ebc5b4047ddf65 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 30 Dec 2015 02:45:47 +0100 Subject: s 1 test-all-krebs-modules: init --- krebs/3modules/shared/default.nix | 1 + shared/1systems/test-all-krebs-modules.nix | 45 ++++++++++++++++++++++++++++++ shared/2configs/buildbot-standalone.nix | 17 ++++++++--- 3 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 shared/1systems/test-all-krebs-modules.nix diff --git a/krebs/3modules/shared/default.nix b/krebs/3modules/shared/default.nix index b332676c6..518e46587 100644 --- a/krebs/3modules/shared/default.nix +++ b/krebs/3modules/shared/default.nix @@ -7,6 +7,7 @@ let "test-arch" "test-centos6" "test-centos7" + "test-all-krebs-modules" ] (name: { inherit name; cores = 1; diff --git a/shared/1systems/test-all-krebs-modules.nix b/shared/1systems/test-all-krebs-modules.nix new file mode 100644 index 000000000..b98004dfe --- /dev/null +++ b/shared/1systems/test-all-krebs-modules.nix @@ -0,0 +1,45 @@ +{ config, pkgs, lib, ... }: +let + en = { enable = true;}; +in { + krebs = { + enable = true; + build.user = config.krebs.users.shared; + build.host = config.krebs.hosts.test-all-krebs-modules; + Reaktor.enable = true; + apt-cacher-ng.enable = true; + backup.enable = true; + bepasty.enable = true; + buildbot.master.enable = true; + buildbot.slave = { + enable = true; + username = "lol"; + password = "wut"; + }; + exim-retiolum.enable = true; + exim-smarthost = { + enable = true; + system-aliases = [ { from = "dick"; to = "butt"; } ]; + }; + go.enable = true; + iptables = { + enable = true; + tables = {}; + }; + nginx.enable = true; + realwallpaper.enable = true; + retiolum.enable = true; + retiolum-bootstrap.enable = true; + tinc_graphs.enable = true; + urlwatch.enable = true; + fetchWallpaper = { + enable = true; + url ="localhost"; + }; + }; + # just get the system running + boot.loader.grub.devices = ["/dev/sda"]; + fileSystems."/" = { + device = "/dev/lol"; + }; +} diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix index 28d1eef2e..22e9861cc 100644 --- a/shared/2configs/buildbot-standalone.nix +++ b/shared/2configs/buildbot-standalone.nix @@ -8,12 +8,9 @@ in { }; networking.firewall.allowedTCPPorts = [ 8010 9989 ]; krebs.buildbot.master = { - secrets = [ - "cac.json" - ]; + secrets = [ "retiolum-ci.rsa_key.priv" "cac.json" ]; slaves = { testslave = "krebspass"; - omo = "krebspass"; }; change_source.stockholm = '' stockholm_repo = 'http://cgit.gum/stockholm' @@ -85,6 +82,18 @@ in { addShell(f,name="eval-cross-check",env=env, command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"]) + addShell(f,name="instaniate-test-all-modules",env=env, + command=nixshell + \ + ["touch retiolum.rsa_key.priv; \ + nix-instantiate --eval -A \ + users.shared.test-all-krebs-modules.system \ + -I stockholm=. \ + -I secrets=. '' \ + --argstr current-date lol \ + --argstr current-user-name shared \ + --argstr current-host-name lol \ + --strict --json"]) + bu.append(util.BuilderConfig(name="fast-tests", slavenames=slavenames, factory=f)) -- cgit v1.2.3 From 2bee6a0f6d71fbb1e1f02bd0df396bc38d3b6f67 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 30 Dec 2015 04:43:16 +0100 Subject: s 2 buildbot-standalone: cosmetics --- shared/2configs/buildbot-standalone.nix | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix index 22e9861cc..c62f8920c 100644 --- a/shared/2configs/buildbot-standalone.nix +++ b/shared/2configs/buildbot-standalone.nix @@ -73,16 +73,16 @@ in { fast-tests = '' f = util.BuildFactory() f.addStep(grab_repo) - addShell(f,name="centos7-eval",env=env, + addShell(f,name="deploy-eval-centos7",env=env, command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"]) - addShell(f,name="wolf-eval",env=env, + addShell(f,name="deploy-eval-wolf",env=env, command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"]) - addShell(f,name="eval-cross-check",env=env, + addShell(f,name="deploy-eval-cross-check",env=env, command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"]) - addShell(f,name="instaniate-test-all-modules",env=env, + addShell(f,name="instantiate-test-all-modules",env=env, command=nixshell + \ ["touch retiolum.rsa_key.priv; \ nix-instantiate --eval -A \ @@ -110,7 +110,7 @@ in { addShell(s, name="infest-cac-centos7",env=env, sigtermTime=60, # SIGTERM 1 minute before SIGKILL - timeout=5400, # 1.5h timeout + timeout=7200, # 2h command=nixshell + ["infest-cac-centos7"]) bu.append(util.BuilderConfig(name="full-tests", @@ -137,6 +137,7 @@ in { username = "testslave"; password = "krebspass"; packages = with pkgs;[ git nix ]; + # all nix commands will need a working nixpkgs installation extraEnviron = { NIX_PATH="nixpkgs=${toString }"; }; }; } -- cgit v1.2.3 From 1798dbaf47fea7793545be2bc78ac5f1c8e27e18 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 30 Dec 2015 04:56:53 +0100 Subject: k 5 test: fix endless loop in test --- krebs/5pkgs/test/infest-cac-centos7/notes | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes index 3f4fcd859..eee0bfc34 100755 --- a/krebs/5pkgs/test/infest-cac-centos7/notes +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -81,9 +81,10 @@ while true;do if ! wait_login_cac $id;then echo "unable to boot a working system within time frame, retrying..." >&2 echo "Cleaning up old image,last status: $(cac update;cac getserver $id | jq -r .status)" - eval "$(clear_defer)" + eval "$(clear_defer | sed 's/;exit//')" else echo "got a working system" >&2 + break fi done clear_defer >/dev/null -- cgit v1.2.3 From c962e8549e968fd15d4f15b4d184e86e1cd7ed04 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 30 Dec 2015 11:29:28 +0100 Subject: k 3 Reaktor: add channels Option --- krebs/3modules/Reaktor.nix | 11 +++++++++-- makefu/1systems/wry.nix | 21 ++++++++++++++++----- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/krebs/3modules/Reaktor.nix b/krebs/3modules/Reaktor.nix index 607eb7cac..92400139c 100644 --- a/krebs/3modules/Reaktor.nix +++ b/krebs/3modules/Reaktor.nix @@ -70,12 +70,17 @@ let REAKTOR_HOST REAKTOR_PORT REAKTOR_STATEDIR - REAKTOR_CHANNELS debug and nickname can be set separately via the Reaktor api ''; }; - + channels = mkOption { + default = [ "#krebs" ]; + type = types.listOf types.str; + description = '' + Channels the Reaktor should connect to at startup. + ''; + }; debug = mkOption { default = false; description = '' @@ -112,7 +117,9 @@ let GIT_SSL_CAINFO = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; REAKTOR_NICKNAME = cfg.nickname; REAKTOR_DEBUG = (if cfg.debug then "True" else "False"); + REAKTOR_CHANNELS = lib.concatStringsSep "," cfg.channels; state_dir = cfg.workdir; + } // cfg.extraEnviron; serviceConfig= { ExecStartPre = pkgs.writeScript "Reaktor-init" '' diff --git a/makefu/1systems/wry.nix b/makefu/1systems/wry.nix index cd2b3f657..3bdf053db 100644 --- a/makefu/1systems/wry.nix +++ b/makefu/1systems/wry.nix @@ -18,8 +18,6 @@ in { ../2configs/iodined.nix - # Reaktor - ../2configs/Reaktor/simpleExtend.nix # other nginx ../2configs/nginx/euer.wiki.nix @@ -29,9 +27,22 @@ in { # collectd ../2configs/collectd/collectd-base.nix ]; + krebs.build.host = config.krebs.hosts.wry; - krebs.Reaktor.enable = true; + krebs.Reaktor = { + nickname = "Reaktor|bot"; + channels = [ "#krebs_test" ]; + enable = true; + debug = true; + plugins = with pkgs.ReaktorPlugins;[ + titlebot + # stockholm-issue + nixos-version + shack-correct + sed-plugin + random-emoji ]; + }; # bepasty to listen only on the correct interfaces krebs.bepasty.servers.internal.nginx.listen = [ "${internal-ip}:80" ]; @@ -59,11 +70,11 @@ in { }; networking = { - firewall = { + firewall = { allowPing = true; logRefusedConnections = false; allowedTCPPorts = [ 53 80 443 ]; - allowedUDPPorts = [ 655 ]; + allowedUDPPorts = [ 655 53 ]; }; interfaces.enp2s1.ip4 = [{ address = external-ip; -- cgit v1.2.3 From f0ce9a72a6595f521f68a156aa46b2372a391d38 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 30 Dec 2015 11:52:22 +0100 Subject: k 5 Reaktor.plugins: fix sed-plugin --- krebs/5pkgs/Reaktor/plugins.nix | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix index 5c7b89f5c..7490be4ca 100644 --- a/krebs/5pkgs/Reaktor/plugins.nix +++ b/krebs/5pkgs/Reaktor/plugins.nix @@ -14,6 +14,7 @@ rec { buildSimpleReaktorPlugin = name: { script , path ? [] , env ? {} + , append_rule ? false # append the rule instead of insert , pattern ? "" , ... } @ attrs: let @@ -26,7 +27,7 @@ rec { }); src_file = "${src_dir}/bin/${name}"; config = '' - public_commands.insert(0,{ + public_commands.${if append_rule then "append(" else "insert(0," }{ 'capname' : "${name}", 'pattern' : ${if pattern == "" then ''indirect_pattern.format("${name}")'' else @@ -58,9 +59,10 @@ rec { }; sed-plugin = buildSimpleReaktorPlugin "sed-plugin" { - path = [ pkgs.gnused ]; + path = [ pkgs.gnused pkgs.python3 ]; # only support s///gi the plugin needs to see every msg # TODO: this will eat up the last regex, fix Reaktor to support fallthru + append_rule = true; pattern = "^(?P.*)$$"; script = ./scripts/sed-plugin.py; }; @@ -105,7 +107,7 @@ rec { config = '' def titlebot_cmd(cmd): from os import environ - return { 'capname': cmd, + return { 'capname': None, 'env': { 'TITLEDB': environ['state_dir']+'/suggestions.json' }, 'pattern': '^\\.' + cmd + '\\s*(?:\\s+(?P.*))?$$', -- cgit v1.2.3 From f7894c29dbfb8404aeb9f4d387942fd638434a22 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 30 Dec 2015 11:53:48 +0100 Subject: m 1 wry: update Reaktor config --- makefu/1systems/wry.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/makefu/1systems/wry.nix b/makefu/1systems/wry.nix index 3bdf053db..f022311c9 100644 --- a/makefu/1systems/wry.nix +++ b/makefu/1systems/wry.nix @@ -32,9 +32,8 @@ in { krebs.Reaktor = { nickname = "Reaktor|bot"; - channels = [ "#krebs_test" ]; + channels = [ "#krebs" "#shackspace" "#binaergewitter" ]; enable = true; - debug = true; plugins = with pkgs.ReaktorPlugins;[ titlebot # stockholm-issue -- cgit v1.2.3 From ca9e1700ef0deac0b71d4c3e2a6d1ee0a0ccbc42 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 30 Dec 2015 14:47:40 +0100 Subject: s 1 minimal-deploy: init test --- shared/1systems/test-minimal-deploy.nix | 13 +++++++++++++ shared/2configs/buildbot-standalone.nix | 11 +++++++++++ 2 files changed, 24 insertions(+) create mode 100644 shared/1systems/test-minimal-deploy.nix diff --git a/shared/1systems/test-minimal-deploy.nix b/shared/1systems/test-minimal-deploy.nix new file mode 100644 index 000000000..ddd96f6b5 --- /dev/null +++ b/shared/1systems/test-minimal-deploy.nix @@ -0,0 +1,13 @@ +{ config, pkgs, lib, ... }: +{ + krebs = { + enable = true; + build.user = config.krebs.users.shared; + build.host = config.krebs.hosts.test-all-krebs-modules; + }; + # just get the system running + boot.loader.grub.devices = ["/dev/sda"]; + fileSystems."/" = { + device = "/dev/lol"; + }; +} diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix index c62f8920c..6ffd7fe8a 100644 --- a/shared/2configs/buildbot-standalone.nix +++ b/shared/2configs/buildbot-standalone.nix @@ -94,6 +94,17 @@ in { --argstr current-host-name lol \ --strict --json"]) + addShell(f,name="instantiate-test-minimal-deploy",env=env, + command=nixshell + \ + ["nix-instantiate --eval -A \ + users.shared.test-minimal-deploy.system \ + -I stockholm=. \ + -I secrets=. '' \ + --argstr current-date lol \ + --argstr current-user-name shared \ + --argstr current-host-name lol \ + --strict --json"]) + bu.append(util.BuilderConfig(name="fast-tests", slavenames=slavenames, factory=f)) -- cgit v1.2.3 From 5d15b95ac200359392d9a86a68905c2162404904 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 30 Dec 2015 23:37:02 +0100 Subject: s 2 buildbot: add short tree timeout before trying a test --- shared/2configs/buildbot-standalone.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix index 6ffd7fe8a..544b54dde 100644 --- a/shared/2configs/buildbot-standalone.nix +++ b/shared/2configs/buildbot-standalone.nix @@ -30,6 +30,7 @@ in { # test the master real quick sched.append(schedulers.SingleBranchScheduler( change_filter=util.ChangeFilter(branch="master"), + treeStableTimer=10, #only test the latest push name="fast-master-test", builderNames=["fast-tests"])) ''; -- cgit v1.2.3 From f916b84ebd6629d7471f50fbb468161285f5026e Mon Sep 17 00:00:00 2001 From: makefu Date: Sat, 2 Jan 2016 17:31:06 +0100 Subject: k default: root path for populate --- krebs/default.nix | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/krebs/default.nix b/krebs/default.nix index 81ddd3ea6..15d0e8e2e 100644 --- a/krebs/default.nix +++ b/krebs/default.nix @@ -50,7 +50,6 @@ let out = { # Prepare target source via bind-mounting - (${populate (args // { infesting = true;}) }) (${nixos-install args}) @@ -103,6 +102,7 @@ let out = { #! /bin/sh # ${current-date} ${current-user-name}@${current-host-name} # krebs.nixos-install + (${populate (args // { root = "/mnt"; })}) ${rootssh target '' export PATH; PATH=/root/.nix-profile/bin:$PATH @@ -209,7 +209,7 @@ let out = { populate = { system ? current-host-name , target ? system - , infesting ? false + , root ? "" }@args: let out = '' #! /bin/sh @@ -223,7 +223,6 @@ let out = { ''; - target_prefix=lib.optionalString infesting "/mnt"; config = get-config system; current-host = config.krebs.hosts.${current-host-name}; @@ -232,7 +231,7 @@ let out = { methods.dir = config: let can-push = config.host.name == current-host.name; - target-path = target_prefix + config.target-path; + target-path = root + config.target-path; push-method = '' rsync \ --exclude .git \ @@ -252,7 +251,7 @@ let out = { throw "No way to push ${dir} from ${current-host.name} to ${target}"; methods.git = config: - let target-path = target_prefix + config.target-path; + let target-path = root + config.target-path; in rootssh target '' mkdir -p ${target-path} cd ${target-path} -- cgit v1.2.3 From 6fb2bff38742607dda99e24ebb40466839e44a16 Mon Sep 17 00:00:00 2001 From: makefu Date: Sat, 2 Jan 2016 21:22:00 +0100 Subject: ma 1 filepimp: add missing kernel modules pata_atiixp is required for booting sata --- makefu/1systems/filepimp.nix | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/makefu/1systems/filepimp.nix b/makefu/1systems/filepimp.nix index 66ea2ce90..1e9ee5031 100644 --- a/makefu/1systems/filepimp.nix +++ b/makefu/1systems/filepimp.nix @@ -17,15 +17,15 @@ loader.grub.device = "/dev/sda"; initrd.availableKernelModules = [ - "usb_storage" "ahci" - "xhci_hcd" - "ata_piix" - "uhci_hcd" + "ohci_pci" "ehci_pci" + "pata_atiixp" + "usb_storage" + "usbhid" ]; - kernelModules = [ ]; + kernelModules = [ "kvm-amd" ]; extraModulePackages = [ ]; }; -- cgit v1.2.3 From f0e802d2593ebe7101968deb3593f1c120f552fd Mon Sep 17 00:00:00 2001 From: makefu Date: Sat, 2 Jan 2016 21:36:51 +0100 Subject: k 5 test/infest-cac-centos7: add timeouts, error handling --- krebs/5pkgs/test/infest-cac-centos7/notes | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes index eee0bfc34..6bfb6906e 100755 --- a/krebs/5pkgs/test/infest-cac-centos7/notes +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -55,9 +55,16 @@ old_trapstr=$(clear_defer) while true;do # Template 26: CentOS7 # TODO: use cac templates to determine the real Centos7 template in case it changes - name=$( cac build cpu=1 ram=512 storage=10 os=26 2>&1\ - | jq -r .servername) - id=servername:$name + out=$(cac build cpu=1 ram=512 storage=10 os=26 2>&1) + if name=$(echo "$out" | jq -r .servername);then + id=servername:$name + echo "got a working machine, id=$id" + else + echo "Unable to build a virtual machine, retrying in 15 seconds" >&2 + echo "Output of build program: $out" >&2 + sleep 15 + continue + fi clear_defer >/dev/null defer "cac delete $id" @@ -65,8 +72,8 @@ while true;do # TODO: timeout? wait_login_cac(){ - # we wait for 15 minutes - for t in `seq 90`;do + # we wait for 30 minutes + for t in `seq 180`;do # now we have a working cac server if cac ssh $1 -o ConnectTimeout=10 \ cat /etc/redhat-release | \ @@ -82,6 +89,7 @@ while true;do echo "unable to boot a working system within time frame, retrying..." >&2 echo "Cleaning up old image,last status: $(cac update;cac getserver $id | jq -r .status)" eval "$(clear_defer | sed 's/;exit//')" + sleep 15 else echo "got a working system" >&2 break -- cgit v1.2.3 From 98848a9fffc8f4a2f456770654648f04bf92d5e2 Mon Sep 17 00:00:00 2001 From: makefu Date: Sun, 3 Jan 2016 06:07:35 +0100 Subject: ma 1 omo: actually build the host --- makefu/1systems/omo.nix | 48 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/makefu/1systems/omo.nix b/makefu/1systems/omo.nix index 6ae79398a..08923d1c2 100644 --- a/makefu/1systems/omo.nix +++ b/makefu/1systems/omo.nix @@ -6,32 +6,64 @@ { imports = - [ # Include the results of the hardware scan. + [ + # TODO: unlock home partition via ssh ../2configs/fs/single-partition-ext4.nix ../2configs/tinc-basic-retiolum.nix + ../2configs/zsh-user.nix ../2configs/exim-retiolum.nix + ../2configs/smart-monitor.nix ]; krebs.build.host = config.krebs.hosts.omo; + services.smartd.devices = [ + { device = "/dev/sda"; } + { device = "/dev/sdb"; } + { device = "/dev/sdc"; } + { device = "/dev/sdd"; } + { device = "/dev/sde"; } + ]; # AMD E350 + fileSystems."/home" = { + device = "/dev/mapper/home"; + fsType = "ext4"; + }; + powerManagement.powerUpCommands = '' + for i in a b c d e f g h i;do + ${pkgs.hdparm}/sbin/hdparm -S 100 /dev/sd$i + ${pkgs.hdparm}/sbin/hdparm -B 127 /dev/sd$i + ${pkgs.hdparm}/sbin/hdparm -y /dev/sd$i + ''; boot = { - loader.grub.device = "/dev/sda"; + initrd.luks = { + devices = [ + { name = "home"; + device = "/dev/disk/by-uuid/85bff22e-dcbb-4246-b030-faf6c1782995"; + keyFileSize = 4096; + keyFile = "/dev/disk/by-id/usb-Verbatim_STORE_N_GO_070B3CEE0B223954-0:0"; } + ]; + }; + loader.grub.device = "/dev/disk/by-id/ata-INTEL_SSDSA2M080G2GC_CVPO003402PB080BGN"; initrd.availableKernelModules = [ - "usb_storage" "ahci" - "xhci_hcd" - "ata_piix" - "uhci_hcd" + "ohci_pci" "ehci_pci" + "pata_atiixp" + "firewire_ohci" + "usb_storage" + "usbhid" ]; - kernelModules = [ ]; + kernelModules = [ "kvm-amd" ]; extraModulePackages = [ ]; }; + networking.firewall.allowedUDPPorts = [ 655 ]; hardware.enableAllFirmware = true; hardware.cpu.amd.updateMicrocode = true; - networking.firewall.allowPing = true; + #zramSwap.enable = true; + zramSwap.numDevices = 2; + } -- cgit v1.2.3 From 757953e551d157b42c06f50e6592cbb3ee64747e Mon Sep 17 00:00:00 2001 From: makefu Date: Sun, 3 Jan 2016 06:08:01 +0100 Subject: ma 1 filepimp: prepare raid --- makefu/1systems/filepimp.nix | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/makefu/1systems/filepimp.nix b/makefu/1systems/filepimp.nix index 1e9ee5031..2d008cee6 100644 --- a/makefu/1systems/filepimp.nix +++ b/makefu/1systems/filepimp.nix @@ -9,12 +9,19 @@ [ # Include the results of the hardware scan. ../2configs/fs/single-partition-ext4.nix ../2configs/tinc-basic-retiolum.nix + ../2configs/smart-monitor.nix ]; krebs.build.host = config.krebs.hosts.filepimp; - + services.smartd.devices = [ + { device = "/dev/sda"; } + { device = "/dev/sdb"; } + { device = "/dev/sdc"; } + { device = "/dev/sdd"; } + { device = "/dev/sde"; } + ]; # AMD N54L boot = { - loader.grub.device = "/dev/sda"; + loader.grub.device = "/dev/sde"; initrd.availableKernelModules = [ "ahci" @@ -28,9 +35,9 @@ kernelModules = [ "kvm-amd" ]; extraModulePackages = [ ]; }; - hardware.enableAllFirmware = true; hardware.cpu.amd.updateMicrocode = true; - networking.firewall.allowPing = true; + zramSwap.enable = true; + zramSwap.numDevices = 2; } -- cgit v1.2.3 From e67393f792d885256456341eee1b9ed21403c01f Mon Sep 17 00:00:00 2001 From: makefu Date: Sun, 3 Jan 2016 06:08:36 +0100 Subject: ma 2 default: bump nixpkgs revision to unstable --- makefu/2configs/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/makefu/2configs/default.nix b/makefu/2configs/default.nix index a0b49edaf..7593eaff7 100644 --- a/makefu/2configs/default.nix +++ b/makefu/2configs/default.nix @@ -23,8 +23,8 @@ with lib; source = { git.nixpkgs = { #url = https://github.com/NixOS/nixpkgs; - url = mkDefault https://github.com/makefu/nixpkgs; - rev = mkDefault "3fd2c24685f604edc925f73ed56600b8c66236b3"; # nixos-15.09 + cherry-picking + url = mkDefault https://github.com/nixos/nixpkgs; + rev = mkDefault "93d8671e2c6d1d25f126ed30e5e6f16764330119"; # unstable @ 2015-01-03, tested on filepimp target-path = "/var/src/nixpkgs"; }; -- cgit v1.2.3 From 1ba7e916206ee1d40a62c13a65f68da5968182a9 Mon Sep 17 00:00:00 2001 From: makefu Date: Sun, 3 Jan 2016 06:09:12 +0100 Subject: ma 2 smartd: enable exim-retiolum by default --- makefu/2configs/smart-monitor.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/makefu/2configs/smart-monitor.nix b/makefu/2configs/smart-monitor.nix index 7086f622b..9b0290a9b 100644 --- a/makefu/2configs/smart-monitor.nix +++ b/makefu/2configs/smart-monitor.nix @@ -1,5 +1,6 @@ -{ config, ... }: +{ config, lib, ... }: { + krebs.exim-retiolum.enable = lib.mkDefault true; services.smartd = { enable = true; notifications = { @@ -11,7 +12,7 @@ # short daily, long weekly, check on boot defaults.monitored = "-a -o on -s (S/../.././02|L/../../7/04)"; - devices = [{ + devices = lib.mkDefault [{ device = "/dev/sda"; }]; }; -- cgit v1.2.3 From 6cb83cd17413be412836041d8235793ff53e66f5 Mon Sep 17 00:00:00 2001 From: makefu Date: Sun, 3 Jan 2016 23:07:55 +0100 Subject: m 1 omo: act as mail client --- makefu/1systems/omo.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/makefu/1systems/omo.nix b/makefu/1systems/omo.nix index 08923d1c2..d7d3dba00 100644 --- a/makefu/1systems/omo.nix +++ b/makefu/1systems/omo.nix @@ -13,6 +13,7 @@ ../2configs/zsh-user.nix ../2configs/exim-retiolum.nix ../2configs/smart-monitor.nix + ../2configs/mail-client.nix ]; krebs.build.host = config.krebs.hosts.omo; services.smartd.devices = [ -- cgit v1.2.3 From d73c8df6e4246f34e7a98091bc3c7dab9f90fdde Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 5 Jan 2016 16:07:13 +0100 Subject: k 5 snapraid: is part of upstream --- krebs/5pkgs/snapraid/default.nix | 33 --------------------------- makefu/1systems/omo.nix | 49 +++++++++++++++++++++++----------------- 2 files changed, 28 insertions(+), 54 deletions(-) delete mode 100644 krebs/5pkgs/snapraid/default.nix diff --git a/krebs/5pkgs/snapraid/default.nix b/krebs/5pkgs/snapraid/default.nix deleted file mode 100644 index 41db0f284..000000000 --- a/krebs/5pkgs/snapraid/default.nix +++ /dev/null @@ -1,33 +0,0 @@ -{stdenv, fetchurl}: -let - s = # Generated upstream information - rec { - baseName="jq"; - version="1.5"; - name="${baseName}-${version}"; - url=https://github.com/stedolan/jq/releases/download/jq-1.5/jq-1.5.tar.gz; - sha256="0g29kyz4ykasdcrb0zmbrp2jqs9kv1wz9swx849i2d1ncknbzln4"; - }; - buildInputs = [ - ]; -in -stdenv.mkDerivation { - inherit (s) name version; - inherit buildInputs; - src = fetchurl { - inherit (s) url sha256; - }; - - # jq is linked to libjq: - configureFlags = [ - "LDFLAGS=-Wl,-rpath,\\\${libdir}" - ]; - meta = { - inherit (s) version; - description = ''A lightweight and flexible command-line JSON processor''; - license = stdenv.lib.licenses.mit ; - maintainers = [stdenv.lib.maintainers.raskin]; - platforms = stdenv.lib.platforms.linux ++ stdenv.lib.platforms.darwin; - }; -} - diff --git a/makefu/1systems/omo.nix b/makefu/1systems/omo.nix index d7d3dba00..65a25a2a1 100644 --- a/makefu/1systems/omo.nix +++ b/makefu/1systems/omo.nix @@ -2,9 +2,18 @@ # your system. Help is available in the configuration.nix(5) man page # and in the NixOS manual (accessible by running ‘nixos-help’). -{ config, pkgs, ... }: - -{ +{ config, pkgs, lib, ... }: +let + byid = dev: "/dev/disk/by-id/" + dev; + keyFile = "/dev/disk/by-id/usb-Verbatim_STORE_N_GO_070B3CEE0B223954-0:0"; + rootDisk = byid "ata-INTEL_SSDSA2M080G2GC_CVPO003402PB080BGN"; + homePartition = byid "ata-INTEL_SSDSA2M080G2GC_CVPO003402PB080BGN-part3"; + cryptDisk0 = byid "ata-ST2000DM001-1CH164_Z240XTT6"; + cryptDisk1 = byid "ata-TP02000GB_TPW151006050068"; + cryptDisk2 = byid "ata-WDC_WD20EARS-00MVWB0_WD-WCAZA5548487"; + # all physical disks + allDisks = [ rootDisk cryptDisk0 cryptDisk1 cryptDisk2 ]; +in { imports = [ # TODO: unlock home partition via ssh @@ -16,35 +25,33 @@ ../2configs/mail-client.nix ]; krebs.build.host = config.krebs.hosts.omo; - services.smartd.devices = [ - { device = "/dev/sda"; } - { device = "/dev/sdb"; } - { device = "/dev/sdc"; } - { device = "/dev/sdd"; } - { device = "/dev/sde"; } - ]; + services.smartd.devices = builtins.map (x: { device = x; }) allDisks; # AMD E350 fileSystems."/home" = { device = "/dev/mapper/home"; fsType = "ext4"; }; - powerManagement.powerUpCommands = '' - for i in a b c d e f g h i;do - ${pkgs.hdparm}/sbin/hdparm -S 100 /dev/sd$i - ${pkgs.hdparm}/sbin/hdparm -B 127 /dev/sd$i - ${pkgs.hdparm}/sbin/hdparm -y /dev/sd$i - ''; + powerManagement.powerUpCommands = lib.concatStrings (map (disk: '' + ${pkgs.hdparm}/sbin/hdparm -S 100 ${disk} + ${pkgs.hdparm}/sbin/hdparm -B 127 ${disk} + ${pkgs.hdparm}/sbin/hdparm -y ${disk} + '') allDisks); boot = { initrd.luks = { - devices = [ - { name = "home"; - device = "/dev/disk/by-uuid/85bff22e-dcbb-4246-b030-faf6c1782995"; + devices = let + usbkey = name: device: { + inherit name device keyFile; keyFileSize = 4096; - keyFile = "/dev/disk/by-id/usb-Verbatim_STORE_N_GO_070B3CEE0B223954-0:0"; } + }; + in [ + (usbkey "home" homePartition) + (usbkey "crypt0" cryptDisk0) + (usbkey "crypt1" cryptDisk1) + (usbkey "crypt2" cryptDisk2) ]; }; - loader.grub.device = "/dev/disk/by-id/ata-INTEL_SSDSA2M080G2GC_CVPO003402PB080BGN"; + loader.grub.device = rootDisk; initrd.availableKernelModules = [ "ahci" -- cgit v1.2.3 From 719b8fb7a8b9b4992200c222b37bd9a6744c25ec Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 5 Jan 2016 16:21:01 +0100 Subject: ma 3 snapraid: init, configuration for omo --- makefu/1systems/omo.nix | 29 ++++++++-- makefu/3modules/default.nix | 1 + makefu/3modules/snapraid.nix | 125 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 5 deletions(-) create mode 100644 makefu/3modules/snapraid.nix diff --git a/makefu/1systems/omo.nix b/makefu/1systems/omo.nix index 65a25a2a1..e19205a95 100644 --- a/makefu/1systems/omo.nix +++ b/makefu/1systems/omo.nix @@ -8,6 +8,10 @@ let keyFile = "/dev/disk/by-id/usb-Verbatim_STORE_N_GO_070B3CEE0B223954-0:0"; rootDisk = byid "ata-INTEL_SSDSA2M080G2GC_CVPO003402PB080BGN"; homePartition = byid "ata-INTEL_SSDSA2M080G2GC_CVPO003402PB080BGN-part3"; + # cryptsetup luksFormat $dev --cipher aes-xts-plain64 -s 512 -h sha512 + # cryptsetup luksAddKey $dev tmpkey + # cryptsetup luksOpen $dev crypt0 + # mkfs.xfs /dev/mapper/crypt0 -L crypt0 cryptDisk0 = byid "ata-ST2000DM001-1CH164_Z240XTT6"; cryptDisk1 = byid "ata-TP02000GB_TPW151006050068"; cryptDisk2 = byid "ata-WDC_WD20EARS-00MVWB0_WD-WCAZA5548487"; @@ -23,15 +27,30 @@ in { ../2configs/exim-retiolum.nix ../2configs/smart-monitor.nix ../2configs/mail-client.nix + ../3modules ]; krebs.build.host = config.krebs.hosts.omo; services.smartd.devices = builtins.map (x: { device = x; }) allDisks; - - # AMD E350 - fileSystems."/home" = { - device = "/dev/mapper/home"; - fsType = "ext4"; + makefu.snapraid = let + toMapper = id: "/media/crypt${builtins.toString id}"; + in { + enable = true; + disks = map toMapper [ 0 1 ]; + parity = toMapper 2; }; + # AMD E350 + fileSystems = let + cryptMount = name: + { "/media/${name}" = { device = "/dev/mapper/${name}"; fsType = "xfs"; };}; + in { + "/home" = { + device = "/dev/mapper/home"; + fsType = "ext4"; + }; + } // cryptMount "crypt0" + // cryptMount "crypt1" + // cryptMount "crypt2"; + powerManagement.powerUpCommands = lib.concatStrings (map (disk: '' ${pkgs.hdparm}/sbin/hdparm -S 100 ${disk} ${pkgs.hdparm}/sbin/hdparm -B 127 ${disk} diff --git a/makefu/3modules/default.nix b/makefu/3modules/default.nix index a8a1f69d0..218c9138e 100644 --- a/makefu/3modules/default.nix +++ b/makefu/3modules/default.nix @@ -2,6 +2,7 @@ _: { imports = [ + ./snapraid.nix ]; } diff --git a/makefu/3modules/snapraid.nix b/makefu/3modules/snapraid.nix new file mode 100644 index 000000000..fbdf50219 --- /dev/null +++ b/makefu/3modules/snapraid.nix @@ -0,0 +1,125 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + # returns dirname without / , used as disk name + dname = dir: replaceChars ["/"] [""] (head (reverseList (splitString "/" dir))); + snapraid-conf = '' + # Disks + ${concatMapStringsSep "\n" (d: "disk ${dname d} ${d}") cfg.disks} + # Parity + ${optionalString (cfg.parity != "") "parity ${cfg.parity}/snapraid.parity"} + + # content on Disks + ${optionalString cfg.contentOnDisks + concatMapStringsSep "\n" (d: "content ${d}/snapraid.content") cfg.disks} + + # content on Parity + ${optionalString (cfg.contentOnParity && cfg.parity != "") + "content ${cfg.parity}/snapraid.content"} + # Default content file + content ${cfg.defaultContentFile} + + # Extra Configuration + ${cfg.extraConfig} + ''; + cfg = config.makefu.snapraid; + + out = { + options.makefu.snapraid = api; + config = mkIf cfg.enable imp; + }; + + api = { + enable = mkEnableOption "snapraid"; + + timerConfig = mkOption { + type = types.unspecified; + description = '' + Start snapraid service + ''; + default = { + OnCalendar = "daily"; + }; + }; + disks = mkOption { + type = with types;listOf str; + description = '' + Disks to protect. Each disk is a path to the mounted directory of the + disk. + ''; + }; + parity = mkOption { + type = types.str; + description = '' + Folder to store parity file. + Set to empty string if you want to configure the parity yourself in + extraConfig. + + All extra parity files (2,3,z, etc...) should be configured via + extraConfig. + ''; + }; + contentOnDisks = mkOption { + type = types.bool; + default = true; + description = '' + Store Content file on each Disk to protect. + Set this to false if you do not want this behavior to apply. + ''; + }; + contentOnParity = mkOption { + type = types.bool; + default = true; + description = '' + Store Content file on parity Disk. + Set this to false if you do not want this behavior to apply. + ''; + }; + defaultContentFile = mkOption { + type = types.str; + default = "/var/cache/snapraid.content"; + description = '' + Path to default content file + Set to empty string if this content file should be written. + ''; + }; + extraConfig = mkOption { + type = types.string; + default = ""; + description = '' + Extra configuration to be appended to the snapraid conf file. + You can configure extra Parity files as well as extra content files. + See `man snapraid` for additional configuration + ''; + }; + }; + + imp = { + environment.systemPackages = [ + # for scrubbing,fixing + pkgs.snapraid + ]; + environment.etc."snapraid.conf".text = snapraid-conf; + systemd.timers.snapraid-sync = { + description = "snapraid sync timer"; + wantedBy = [ "timers.target" ]; + timerConfig = cfg.timerConfig; + }; + systemd.services.snapraid-sync = { + description = "Snapraid sync service"; + after = [ "network.target" "local-fs.target" ]; + + serviceConfig = { + Type = "simple"; + ExecStartPre = pkgs.writeScript "Snapraid-sync-init" '' + #! /bin/sh + ${optionalString (cfg.defaultContentFile != "") + "mkdir -p $(dirname ${cfg.defaultContentFile})"} + ''; + ExecStart = "${pkgs.snapraid}/bin/snapraid sync"; + }; + }; + }; +in out -- cgit v1.2.3 From 1fda893916e1cf8c3cecd43fd861c9d36999b280 Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 5 Jan 2016 16:21:23 +0100 Subject: ma 2 mail-client: put imapfilter,gnupg into the loop --- makefu/2configs/mail-client.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/makefu/2configs/mail-client.nix b/makefu/2configs/mail-client.nix index a6ae33d2f..bda21e9d0 100644 --- a/makefu/2configs/mail-client.nix +++ b/makefu/2configs/mail-client.nix @@ -7,6 +7,8 @@ with lib; mutt-kz notmuch offlineimap + imapfilter + gnupg ]; } -- cgit v1.2.3 From ff945f40b1c3fdb69a5016911ab48462e00cf536 Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 7 Jan 2016 08:05:05 +0100 Subject: s 2 buildbot: up cac timeout to 3h --- shared/2configs/buildbot-standalone.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix index 544b54dde..3275189a5 100644 --- a/shared/2configs/buildbot-standalone.nix +++ b/shared/2configs/buildbot-standalone.nix @@ -122,7 +122,7 @@ in { addShell(s, name="infest-cac-centos7",env=env, sigtermTime=60, # SIGTERM 1 minute before SIGKILL - timeout=7200, # 2h + timeout=10800, # 3h command=nixshell + ["infest-cac-centos7"]) bu.append(util.BuilderConfig(name="full-tests", -- cgit v1.2.3 From 49b6fd9c87678893ed47794b116660700994b1bc Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 7 Jan 2016 17:34:56 +0100 Subject: ma 1 pnp: be able to build as vm --- makefu/1systems/pnp.nix | 64 ++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 36 deletions(-) diff --git a/makefu/1systems/pnp.nix b/makefu/1systems/pnp.nix index a1b73c0c9..51c124bbe 100644 --- a/makefu/1systems/pnp.nix +++ b/makefu/1systems/pnp.nix @@ -1,59 +1,51 @@ -# Edit this configuration file to define what should be installed on -# your system. Help is available in the configuration.nix(5) man page -# and in the NixOS manual (accessible by running ‘nixos-help’). - +# Usage: +# NIX_PATH=secrets=/home/makefu/secrets/wry:nixpkgs=/var/src/nixpkgs nix-build -A users.makefu.pnp.config.system.build.vm +# result/bin/run-pnp-vm -virtfs local,path=/home/makefu/secrets/pnp,security_model=none,mount_tag=secrets { config, pkgs, ... }: { imports = - [ # Include the results of the hardware scan. - # Base + [ ../2configs/tinc-basic-retiolum.nix ../2configs/headless.nix + ../../krebs/3modules/Reaktor.nix - # HW/FS - - # enables virtio kernel modules in initrd + # these will be overwritten by qemu-vm.nix but will be used if the system + # is directly deployed ../2configs/fs/vm-single-partition.nix - # Services - ../2configs/git/cgit-retiolum.nix - - ## Reaktor - ## \/ are only plugins, must enable Reaktor explicitly - ../2configs/Reaktor/stockholmLentil.nix - ../2configs/Reaktor/simpleExtend.nix - ../2configs/Reaktor/random-emoji.nix - ../2configs/Reaktor/titlebot.nix - ../2configs/Reaktor/shack-correct.nix - - # ../2configs/graphite-standalone.nix + # config.system.build.vm + ]; - krebs.urlwatch.verbose = true; - krebs.Reaktor.enable = true; - krebs.Reaktor.debug = true; - krebs.Reaktor.nickname = "Reaktor|bot"; - krebs.Reaktor.extraEnviron = { - REAKTOR_CHANNELS = "#krebs,#binaergewitter,#shackspace"; + virtualisation.graphics = false; + # also export secrets, see Usage above + fileSystems = pkgs.lib.mkVMOverride { + "${builtins.toString }" = + { device = "secrets"; + fsType = "9p"; + options = "trans=virtio,version=9p2000.L,cache=loose"; + neededForBoot = true; + }; + }; + + krebs.Reaktor = { + enable = true; + debug = true; + extraEnviron = { + REAKTOR_HOST = "cd.retiolum"; + }; + plugins = with pkgs.ReaktorPlugins; [ stockholm-issue nixos-version sed-plugin ]; + channels = [ "#retiolum" ]; }; krebs.build.host = config.krebs.hosts.pnp; nixpkgs.config.packageOverrides = pkgs: { tinc = pkgs.tinc_pre; }; - networking.firewall.allowedTCPPorts = [ - # nginx runs on 80 - 80 - # graphite-web runs on 8080, carbon cache runs on 2003 tcp and udp - # 8080 2003 - - # smtp 25 ]; - # networking.firewall.allowedUDPPorts = [ 2003 ]; - } -- cgit v1.2.3 [cgit] Unable to lock slot /tmp/cgit/f1200000.lock: No such file or directory (2)