From 56e8346faa75fc42f65d11ea3569a3e5bdd252ec Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 22 Dec 2015 20:53:11 +0100 Subject: k 5 krebs-ci: remove obsolete trap rm --- krebs/5pkgs/krebs-ci/notes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'krebs/5pkgs') diff --git a/krebs/5pkgs/krebs-ci/notes b/krebs/5pkgs/krebs-ci/notes index 7e34d6a28..f6b193ddb 100755 --- a/krebs/5pkgs/krebs-ci/notes +++ b/krebs/5pkgs/krebs-ci/notes @@ -19,7 +19,7 @@ krebs_secrets=$(mktemp -d) sec_file=$krebs_secrets/cac_config krebs_ssh=$krebs_secrets/tempssh # we need to receive this key from buildmaster to speed up tinc bootstrap -TRAP="rm $sec_file;rm -r $krebs_secrets" +TRAP="rm -r $krebs_secrets" trap "$TRAP" INT TERM EXIT cat > $sec_file < Date: Tue, 22 Dec 2015 23:37:12 +0100 Subject: k 5 krebs-ci: set cache files manually --- krebs/5pkgs/krebs-ci/notes | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'krebs/5pkgs') diff --git a/krebs/5pkgs/krebs-ci/notes b/krebs/5pkgs/krebs-ci/notes index f6b193ddb..f6f3da8db 100755 --- a/krebs/5pkgs/krebs-ci/notes +++ b/krebs/5pkgs/krebs-ci/notes @@ -1,9 +1,10 @@ #! /bin/sh # nix-shell -p gnumake jq openssh cac cacpanel -set -euf +set -eufx # 2 secrets are required: + krebs_cred=${krebs_cred-./cac.json} retiolum_key=${retiolum_key-./retiolum.rsa_key.priv} @@ -18,8 +19,12 @@ fi krebs_secrets=$(mktemp -d) sec_file=$krebs_secrets/cac_config krebs_ssh=$krebs_secrets/tempssh +cac_resources_cache=$krebs_secrets/res_cache.json +cac_servers_cache=$krebs_secrets/servers_cache.json +cac_tasks_cache=$krebs_secrets/tasks_cache.json +cac_templates_cache=$krebs_secrets/templates_cache.json # we need to receive this key from buildmaster to speed up tinc bootstrap -TRAP="rm -r $krebs_secrets" +TRAP="rm -r $krebs_secrets;exit" trap "$TRAP" INT TERM EXIT cat > $sec_file <&1\ id=servername:$name trap "cac delete $id;$TRAP" INT TERM EXIT # TODO: timeout? -always_update=true cac waitstatus $id "Powered On" +# cac_always_update=true cac waitstatus $id "Powered On" wait_login_cac(){ # timeout - for t in `seq 60`;do + for t in `seq 180`;do # now we have a working cac server if cac ssh $1 cat /etc/redhat-release | \ grep CentOS ;then -- cgit v1.3.1 From dc8e270d2a5346e4316b7c2050b26fd428ec3fc3 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 23 Dec 2015 00:06:27 +0100 Subject: m 3 buildbot: add new slow factory to complete integration test --- krebs/3modules/buildbot/master.nix | 52 +++++++++++++++++++++++++------------- krebs/3modules/buildbot/slave.nix | 1 + krebs/5pkgs/krebs-ci/notes | 8 +++--- 3 files changed, 40 insertions(+), 21 deletions(-) (limited to 'krebs/5pkgs') diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index e66e0d6b2..b4fd6bb2f 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -59,27 +59,28 @@ let ###### The actual build # couple of fast steps: f = util.BuildFactory() + # some slow steps + s = util.BuildFactory() ## fetch repo grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') f.addStep(grab_repo) + s.addStep(grab_repo) # the dependencies which are used by the test script - deps = [ "gnumake", "jq" ] - nixshell = ["nix-shell", "-p" ] + deps + [ "--run" ] + deps = [ "gnumake", "jq", "(import {}).pkgs.krebs-ci" ] + nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ] + def addShell(f,**kwargs): f.addStep(steps.ShellCommand(**kwargs)) - addShell(f,name="centos7-eval",env={"LOGNAME": "shared", - "get" : "krebs.deploy", - "filter" : "json" - }, - command=nixshell + ["make -s eval system=test-centos7"]) + addShell(f,name="centos7-eval",env={"LOGNAME": "shared"}, + command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"]) + + addShell(f,name="wolf-eval",env={"LOGNAME": "shared"}, + command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"]) - addShell(f,name="wolf-eval",env={"LOGNAME": "shared", - "get" : "krebs.deploy", - "filter" : "json" - }, - command=nixshell + ["make -s eval system=wolf"]) + addShell(f,name="eval-cross-check",env={"LOGNAME": "shared"}, + command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"]) c['builders'] = [] c['builders'].append( @@ -87,11 +88,20 @@ let slavenames=slavenames, factory=f)) - # TODO slow build + # slave needs 2 files: + # * cac.json + # * retiolum + for file in ["cac.json", "retiolum.rsa_key.priv"]: + s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file), + slavedest=file)) + + addShell(s,name="complete-build-centos7",env={"LOGNAME": "shared"}, + command=nixshell + ["krebs-ci"]) + c['builders'].append( util.BuilderConfig(name="full-tests", slavenames=slavenames, - factory=f)) + factory=s)) ####### Status of Builds c['status'] = [] @@ -106,7 +116,7 @@ let forceBuild = 'auth', forceAllBuilds = 'auth', pingBuilder = False, - stopBuild = False, + stopBuild = 'auth', stopAllBuilds = False, cancelPendingBuild = False, ) @@ -119,8 +129,8 @@ let # TODO: multiple channels channels=["${cfg.irc.channel}"], notify_events={ - #'success': 1, - #'failure': 1, + 'success': 1, + 'failure': 1, 'exception': 1, 'successToFailure': 1, 'failureToSuccess': 1, @@ -219,8 +229,12 @@ let after = [ "network.target" ]; wantedBy = [ "multi-user.target" ]; path = [ pkgs.git ]; + environment = { + SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; + }; serviceConfig = let workdir="${lib.shell.escape cfg.workDir}"; + secretsdir="${lib.shell.escape (toString )}"; # TODO: check if git is the only dep in { PermissionsStartOnly = true; @@ -236,6 +250,10 @@ let fi # always override the master.cfg cp ${buildbot-master-config} ${workdir}/master.cfg + # copy secrets + cp ${secretsdir}/cac.json ${workdir} + cp ${secretsdir}/retiolum-ci.rsa_key.priv \ + ${workdir}/retiolum.rsa_key.priv # sanity ${buildbot}/bin/buildbot checkconfig ${workdir} diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix index 65291f63e..8711a287a 100644 --- a/krebs/3modules/buildbot/slave.nix +++ b/krebs/3modules/buildbot/slave.nix @@ -144,6 +144,7 @@ let path = default-packages ++ cfg.packages; environment = { + SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; NIX_REMOTE="daemon"; } // cfg.extraEnviron; diff --git a/krebs/5pkgs/krebs-ci/notes b/krebs/5pkgs/krebs-ci/notes index f6f3da8db..f162656f7 100755 --- a/krebs/5pkgs/krebs-ci/notes +++ b/krebs/5pkgs/krebs-ci/notes @@ -19,10 +19,10 @@ fi krebs_secrets=$(mktemp -d) sec_file=$krebs_secrets/cac_config krebs_ssh=$krebs_secrets/tempssh -cac_resources_cache=$krebs_secrets/res_cache.json -cac_servers_cache=$krebs_secrets/servers_cache.json -cac_tasks_cache=$krebs_secrets/tasks_cache.json -cac_templates_cache=$krebs_secrets/templates_cache.json +export cac_resources_cache=$krebs_secrets/res_cache.json +export cac_servers_cache=$krebs_secrets/servers_cache.json +export cac_tasks_cache=$krebs_secrets/tasks_cache.json +export cac_templates_cache=$krebs_secrets/templates_cache.json # we need to receive this key from buildmaster to speed up tinc bootstrap TRAP="rm -r $krebs_secrets;exit" trap "$TRAP" INT TERM EXIT -- cgit v1.3.1 From 14ddb767eb10dbe43d3112c4b4674f6c1d4ff32a Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 23 Dec 2015 11:18:00 +0100 Subject: k 5 mv krebs-ci test/infest-cac-centos7 --- krebs/3modules/buildbot/master.nix | 6 +- krebs/5pkgs/default.nix | 4 + krebs/5pkgs/krebs-ci/default.nix | 37 -------- krebs/5pkgs/krebs-ci/notes | 116 ----------------------- krebs/5pkgs/test/infest-cac-centos7/default.nix | 39 ++++++++ krebs/5pkgs/test/infest-cac-centos7/notes | 117 ++++++++++++++++++++++++ 6 files changed, 163 insertions(+), 156 deletions(-) delete mode 100644 krebs/5pkgs/krebs-ci/default.nix delete mode 100755 krebs/5pkgs/krebs-ci/notes create mode 100644 krebs/5pkgs/test/infest-cac-centos7/default.nix create mode 100755 krebs/5pkgs/test/infest-cac-centos7/notes (limited to 'krebs/5pkgs') diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index b4fd6bb2f..483ba18e7 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -67,7 +67,7 @@ let s.addStep(grab_repo) # the dependencies which are used by the test script - deps = [ "gnumake", "jq", "(import {}).pkgs.krebs-ci" ] + deps = [ "gnumake", "jq", "(import {}).pkgs.test.infest-cac-centos7" ] nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ] def addShell(f,**kwargs): @@ -95,8 +95,8 @@ let s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file), slavedest=file)) - addShell(s,name="complete-build-centos7",env={"LOGNAME": "shared"}, - command=nixshell + ["krebs-ci"]) + addShell(s,name="infest-cac-centos7",env={"LOGNAME": "shared"}, + command=nixshell + ["infest-cac-centos7"]) c['builders'].append( util.BuilderConfig(name="full-tests", diff --git a/krebs/5pkgs/default.nix b/krebs/5pkgs/default.nix index 7df7b7d3c..0562fe836 100644 --- a/krebs/5pkgs/default.nix +++ b/krebs/5pkgs/default.nix @@ -40,6 +40,10 @@ subdirs // rec { } ''; + test = { + infest-cac-centos7 = pkgs.callPackage ./test/infest-cac-centos7 {}; + }; + execveBin = name: cfg: execve name (cfg // { destination = "/bin/${name}"; }); writeC = name: { destination ? "" }: src: pkgs.runCommand name {} '' diff --git a/krebs/5pkgs/krebs-ci/default.nix b/krebs/5pkgs/krebs-ci/default.nix deleted file mode 100644 index f5b302b52..000000000 --- a/krebs/5pkgs/krebs-ci/default.nix +++ /dev/null @@ -1,37 +0,0 @@ -{ stdenv, coreutils,makeWrapper, cac, cacpanel, gnumake, gnused, jq, openssh, ... }: - -stdenv.mkDerivation rec { - name = "krebs-ci-0.1.0"; - - src = ./notes; - - phases = [ - "installPhase" - ]; - buildInputs = [ makeWrapper ]; - - path = stdenv.lib.makeSearchPath "bin" [ - coreutils - cac - cacpanel - gnumake - gnused - jq - openssh - ]; - - installPhase = - '' - mkdir -p $out/bin - cp ${src} $out/bin/krebs-ci - chmod +x $out/bin/krebs-ci - wrapProgram $out/bin/krebs-ci \ - --prefix PATH : ${path} - ''; - meta = with stdenv.lib; { - homepage = http://krebsco.de; - description = "Krebs CI Scripts"; - license = licenses.wtfpl; - maintainers = [ maintainers.makefu ]; - }; -} diff --git a/krebs/5pkgs/krebs-ci/notes b/krebs/5pkgs/krebs-ci/notes deleted file mode 100755 index f162656f7..000000000 --- a/krebs/5pkgs/krebs-ci/notes +++ /dev/null @@ -1,116 +0,0 @@ -#! /bin/sh - -# nix-shell -p gnumake jq openssh cac cacpanel -set -eufx - -# 2 secrets are required: - -krebs_cred=${krebs_cred-./cac.json} -retiolum_key=${retiolum_key-./retiolum.rsa_key.priv} - -# Sanity -if test ! -r "$krebs_cred";then - echo "\$krebs_cred=$krebs_cred must be readable"; exit 1 -fi -if test ! -r "$retiolum_key";then - echo "\$retiolum_key=$retiolum_key must be readable"; exit 1 -fi - -krebs_secrets=$(mktemp -d) -sec_file=$krebs_secrets/cac_config -krebs_ssh=$krebs_secrets/tempssh -export cac_resources_cache=$krebs_secrets/res_cache.json -export cac_servers_cache=$krebs_secrets/servers_cache.json -export cac_tasks_cache=$krebs_secrets/tasks_cache.json -export cac_templates_cache=$krebs_secrets/templates_cache.json -# we need to receive this key from buildmaster to speed up tinc bootstrap -TRAP="rm -r $krebs_secrets;exit" -trap "$TRAP" INT TERM EXIT - -cat > $sec_file <&1\ - | jq -r .servername) - -id=servername:$name -trap "cac delete $id;$TRAP" INT TERM EXIT -# TODO: timeout? -# cac_always_update=true cac waitstatus $id "Powered On" - -wait_login_cac(){ - # timeout - for t in `seq 180`;do - # now we have a working cac server - if cac ssh $1 cat /etc/redhat-release | \ - grep CentOS ;then - return 0 - fi - sleep 10 - done - return 1 -} -# die on timeout -wait_login_cac $id - -mkdir -p shared/2configs/temp -cac generatenetworking $id > \ - shared/2configs/temp/networking.nix -# new temporary ssh key we will use to log in after infest -ssh-keygen -f $krebs_ssh -N "" -cp $retiolum_key $krebs_secrets/retiolum.rsa_key.priv -# we override the directories for secrets and stockholm -# additionally we set the ssh key we generated -ip=$(cac getserver $id | jq -r .ip) - -cat > shared/2configs/temp/dirs.nix < $krebs_secrets/infest -sh -x $krebs_secrets/infest - -# TODO: generate secrets directory $krebs_secrets for nix import -cac powerop $id reset - -wait_login(){ - # timeout - for t in `seq 20`;do - # now we have a working cac server - if ssh -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -i $krebs_ssh \ - -o ConnectTimeout=10 \ - -o BatchMode=yes \ - root@$1 nixos-version ;then - return 0 - fi - sleep 10 - done - return 1 -} -wait_login $ip diff --git a/krebs/5pkgs/test/infest-cac-centos7/default.nix b/krebs/5pkgs/test/infest-cac-centos7/default.nix new file mode 100644 index 000000000..7f2e3f231 --- /dev/null +++ b/krebs/5pkgs/test/infest-cac-centos7/default.nix @@ -0,0 +1,39 @@ +{ stdenv, coreutils,makeWrapper, cac, cacpanel, gnumake, gnused, jq, openssh, ... }: + +stdenv.mkDerivation rec { + name = "${shortname}-${version}"; + shortname = "infest-cac-centos7"; + version = "0.2.0"; + + src = ./notes; + + phases = [ + "installPhase" + ]; + buildInputs = [ makeWrapper ]; + + path = stdenv.lib.makeSearchPath "bin" [ + coreutils + cac + cacpanel + gnumake + gnused + jq + openssh + ]; + + installPhase = + '' + mkdir -p $out/bin + cp ${src} $out/bin/${shortname} + chmod +x $out/bin/${shortname} + wrapProgram $out/bin/${shortname} \ + --prefix PATH : ${path} + ''; + meta = with stdenv.lib; { + homepage = http://krebsco.de; + description = "Krebs CI Scripts"; + license = licenses.wtfpl; + maintainers = [ maintainers.makefu ]; + }; +} diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes new file mode 100755 index 000000000..1e350084c --- /dev/null +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -0,0 +1,117 @@ +#! /bin/sh + +# nix-shell -p gnumake jq openssh cac cacpanel +set -eufx + +# 2 secrets are required: + +krebs_cred=${krebs_cred-./cac.json} +retiolum_key=${retiolum_key-./retiolum.rsa_key.priv} + +# Sanity +if test ! -r "$krebs_cred";then + echo "\$krebs_cred=$krebs_cred must be readable"; exit 1 +fi +if test ! -r "$retiolum_key";then + echo "\$retiolum_key=$retiolum_key must be readable"; exit 1 +fi + +krebs_secrets=$(mktemp -d) +sec_file=$krebs_secrets/cac_config +krebs_ssh=$krebs_secrets/tempssh +export cac_resources_cache=$krebs_secrets/res_cache.json +export cac_servers_cache=$krebs_secrets/servers_cache.json +export cac_tasks_cache=$krebs_secrets/tasks_cache.json +export cac_templates_cache=$krebs_secrets/templates_cache.json +# we need to receive this key from buildmaster to speed up tinc bootstrap +TRAP="rm -r $krebs_secrets;trap - INT TERM EXIT" +trap "$TRAP" INT TERM EXIT + +cat > $sec_file <&1\ + | jq -r .servername) + +id=servername:$name +trap "cac delete $id;$TRAP;exit" INT TERM EXIT +# TODO: timeout? + +wait_login_cac(){ + # timeout + for t in `seq 180`;do + # now we have a working cac server + if cac ssh $1 -o ConnectTimeout=10 \ + -o BatchMode=yes \ + cat /etc/redhat-release | \ + grep CentOS ;then + return 0 + fi + sleep 10 + done + return 1 +} +# die on timeout +wait_login_cac $id + +mkdir -p shared/2configs/temp +cac generatenetworking $id > \ + shared/2configs/temp/networking.nix +# new temporary ssh key we will use to log in after infest +ssh-keygen -f $krebs_ssh -N "" +cp $retiolum_key $krebs_secrets/retiolum.rsa_key.priv +# we override the directories for secrets and stockholm +# additionally we set the ssh key we generated +ip=$(cac getserver $id | jq -r .ip) + +cat > shared/2configs/temp/dirs.nix < $krebs_secrets/infest +sh -x $krebs_secrets/infest + +# TODO: generate secrets directory $krebs_secrets for nix import +cac powerop $id reset + +wait_login(){ + # timeout + for t in `seq 20`;do + # now we have a working cac server + if ssh -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -i $krebs_ssh \ + -o ConnectTimeout=10 \ + -o BatchMode=yes \ + root@$1 nixos-version ;then + return 0 + fi + sleep 10 + done + return 1 +} +wait_login $ip -- cgit v1.3.1 From cf3391704d88c49afba652715e1153888bf46099 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 23 Dec 2015 16:02:58 +0100 Subject: k 5 test/infest*: remove batch mode from cac ssh call this leads to "permission denied" --- krebs/5pkgs/test/infest-cac-centos7/notes | 1 - 1 file changed, 1 deletion(-) (limited to 'krebs/5pkgs') diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes index 1e350084c..5fd0cae61 100755 --- a/krebs/5pkgs/test/infest-cac-centos7/notes +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -53,7 +53,6 @@ wait_login_cac(){ for t in `seq 180`;do # now we have a working cac server if cac ssh $1 -o ConnectTimeout=10 \ - -o BatchMode=yes \ cat /etc/redhat-release | \ grep CentOS ;then return 0 -- cgit v1.3.1 From 6e4351044195f1f3b5708785e760b9d118e2c229 Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 24 Dec 2015 02:20:24 +0100 Subject: k 5 test/infest*: up limit of final connect --- krebs/5pkgs/test/infest-cac-centos7/notes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'krebs/5pkgs') diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes index 5fd0cae61..5bb5de2c4 100755 --- a/krebs/5pkgs/test/infest-cac-centos7/notes +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -99,7 +99,7 @@ cac powerop $id reset wait_login(){ # timeout - for t in `seq 20`;do + for t in `seq 90`;do # now we have a working cac server if ssh -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ -- cgit v1.3.1 From 9ae664209328f6030bf3773e09dce7bcd14e82b4 Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 24 Dec 2015 20:37:04 +0100 Subject: k 5 cacpanel: bump version to 0.2.3 --- krebs/5pkgs/cacpanel/default.nix | 4 ++-- krebs/5pkgs/test/infest-cac-centos7/notes | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'krebs/5pkgs') diff --git a/krebs/5pkgs/cacpanel/default.nix b/krebs/5pkgs/cacpanel/default.nix index 3e3e2e1fc..3df4dffed 100644 --- a/krebs/5pkgs/cacpanel/default.nix +++ b/krebs/5pkgs/cacpanel/default.nix @@ -2,11 +2,11 @@ python3Packages.buildPythonPackage rec { name = "cacpanel-${version}"; - version = "0.2.1"; + version = "0.2.3"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/source/c/cacpanel/cacpanel-${version}.tar.gz"; - sha256 = "1zaazg5r10kgva32zh4fhpw6l6h51ijkwpa322na0kh4x6f6aqj3"; + sha256 = "1fib7416qqv8yzrj75kxra7ccpz9abqh58b6gkaavws2fa6m3mm8"; }; propagatedBuildInputs = with python3Packages; [ diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes index 5bb5de2c4..cfb074423 100755 --- a/krebs/5pkgs/test/infest-cac-centos7/notes +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -29,11 +29,11 @@ trap "$TRAP" INT TERM EXIT cat > $sec_file < Date: Fri, 25 Dec 2015 00:04:52 +0100 Subject: k 5 Reaktor: init plugin infrastructure --- krebs/5pkgs/Reaktor/plugins.nix | 38 +++++++++++++++++++++++++++++ krebs/5pkgs/Reaktor/scripts/random-emoji.sh | 6 +++++ krebs/5pkgs/default.nix | 2 ++ 3 files changed, 46 insertions(+) create mode 100644 krebs/5pkgs/Reaktor/plugins.nix create mode 100644 krebs/5pkgs/Reaktor/scripts/random-emoji.sh (limited to 'krebs/5pkgs') diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix new file mode 100644 index 000000000..05ede38e1 --- /dev/null +++ b/krebs/5pkgs/Reaktor/plugins.nix @@ -0,0 +1,38 @@ +{ stdenv, lib, pkgs, makeWrapper }: + +rec { + buildReaktorPlugin = { name + # TODO: profiles + , extraConfig + , phases ? [] + , ... } @ attrs: + stdenv.mkDerivation (attrs // { + name = "Reaktor-plugin-" + name; + phases = phases ++ [ "installPhase" ]; + isReaktorPlugin = true; + }); + + random-emoji = buildReaktorPlugin rec { + name = "random-emoji"; + src = ./scripts/random-emoji.sh; + phases = [ "installPhase" ]; + buildInputs = [ makeWrapper ]; + installPhase = '' + mkdir -p $out/bin + install -vm 755 ${src} $out/bin/random-emoji.sh + wrapProgram $out/bin/random-emoji.sh \ + --prefix PATH : ${lib.makeSearchPath "bin" (with pkgs; [ + coreutils + gnused + gnugrep + xmlstarlet + curl])}; + ''; + extraConfig = '' + public_commands.insert(0,{ + 'capname' : "emoji", + 'pattern' : indirect_pattern.format("emoji"), + 'argv' : ["random-emoji.sh"]) + ''; + }; +} diff --git a/krebs/5pkgs/Reaktor/scripts/random-emoji.sh b/krebs/5pkgs/Reaktor/scripts/random-emoji.sh new file mode 100644 index 000000000..386aa68b9 --- /dev/null +++ b/krebs/5pkgs/Reaktor/scripts/random-emoji.sh @@ -0,0 +1,6 @@ +#!/bin/sh +curl http://emojicons.com/random -s | \ + grep data-text | \ + sed -n 's/.*>\(.*\)<\/textarea>/\1/p' | \ + head -n 1 | \ + xmlstarlet unesc diff --git a/krebs/5pkgs/default.nix b/krebs/5pkgs/default.nix index 0562fe836..c4b1dafe4 100644 --- a/krebs/5pkgs/default.nix +++ b/krebs/5pkgs/default.nix @@ -26,6 +26,8 @@ subdirs // rec { inherit (subdirs) get jq; }; + ReaktorPlugins = pkgs.callPackage ./Reaktor/plugins.nix {}; + execve = name: { filename, argv, envp ? {}, destination ? "" }: writeC name { inherit destination; } '' #include -- cgit v1.3.1 From 763f0db52ad45eef6e09d7982cd0f6cd898857e3 Mon Sep 17 00:00:00 2001 From: tv Date: Sat, 26 Dec 2015 05:55:13 +0100 Subject: {pkgs => lib}.genid --- krebs/3modules/Reaktor.nix | 20 ++++--------------- krebs/3modules/apt-cacher-ng.nix | 6 ++---- krebs/3modules/bepasty-server.nix | 4 ++-- krebs/3modules/fetchWallpaper.nix | 2 +- krebs/3modules/git.nix | 8 ++++---- krebs/3modules/github-hosts-sync.nix | 4 ++-- krebs/3modules/go.nix | 5 ++--- krebs/3modules/realwallpaper.nix | 10 ++-------- krebs/3modules/retiolum.nix | 4 ++-- krebs/3modules/tinc_graphs.nix | 2 +- krebs/3modules/urlwatch.nix | 5 ++--- krebs/4lib/default.nix | 3 +++ krebs/4lib/genid.nix | 37 ++++++++++++++++++++++++++++++++++++ krebs/5pkgs/genid/default.nix | 22 --------------------- lass/1systems/mors.nix | 1 - lass/2configs/libvirt.nix | 4 ++-- lass/2configs/skype.nix | 4 ++-- lass/2configs/weechat.nix | 2 +- lass/3modules/newsbot-js.nix | 2 +- lass/3modules/owncloud_nginx.nix | 2 +- lass/3modules/wordpress_nginx.nix | 2 +- makefu/1systems/vbob.nix | 1 - makefu/3modules/buildbot/master.nix | 2 +- makefu/3modules/buildbot/slave.nix | 2 +- tv/1systems/wu.nix | 1 - tv/1systems/xu.nix | 1 - tv/2configs/charybdis.nix | 4 ++-- tv/2configs/pulse.nix | 8 +++----- tv/3modules/consul.nix | 4 ++-- tv/3modules/ejabberd.nix | 4 ++-- 30 files changed, 83 insertions(+), 93 deletions(-) create mode 100644 krebs/4lib/genid.nix delete mode 100644 krebs/5pkgs/genid/default.nix (limited to 'krebs/5pkgs') diff --git a/krebs/3modules/Reaktor.nix b/krebs/3modules/Reaktor.nix index 1ec49b81e..0fca52203 100644 --- a/krebs/3modules/Reaktor.nix +++ b/krebs/3modules/Reaktor.nix @@ -1,19 +1,8 @@ -{ config, pkgs,lib, ... }: - +{ config, lib, pkgs, ... }: +with lib; let - inherit (lib) - mkIf - mkOption - types - singleton - isString - optionalString - concatStrings - escapeShellArg - ; - ReaktorConfig = pkgs.writeText "config.py" '' ${if (isString cfg.overrideConfig ) then '' # Overriden Config @@ -86,10 +75,9 @@ let imp = { # for reaktor get-config - users.extraUsers = singleton { + users.extraUsers = singleton rec { name = "Reaktor"; - # uid = config.ids.uids.Reaktor; - uid = 2066439104; #genid Reaktor + uid = genid name; description = "Reaktor user"; home = "/var/lib/Reaktor"; createHome = true; diff --git a/krebs/3modules/apt-cacher-ng.nix b/krebs/3modules/apt-cacher-ng.nix index 75296bafb..371d39b6f 100644 --- a/krebs/3modules/apt-cacher-ng.nix +++ b/krebs/3modules/apt-cacher-ng.nix @@ -119,16 +119,14 @@ let imp = { users.extraUsers.acng = { - # uid = config.ids.uids.acng; - uid = 897955083; #genid Reaktor + uid = genid "acng"; description = "apt-cacher-ng"; home = acng-home; createHome = false; }; users.extraGroups.acng = { - gid = 897955083; #genid Reaktor - # gid = config.ids.gids.Reaktor; + gid = genid "acng"; }; systemd.services.apt-cacher-ng = { diff --git a/krebs/3modules/bepasty-server.nix b/krebs/3modules/bepasty-server.nix index c99c3d11a..e74841205 100644 --- a/krebs/3modules/bepasty-server.nix +++ b/krebs/3modules/bepasty-server.nix @@ -130,12 +130,12 @@ let ) cfg.servers; users.extraUsers.bepasty = { - uid = 2796546855; #genid bepasty + uid = genid "bepasty"; group = "bepasty"; home = "/var/lib/bepasty-server"; }; users.extraGroups.bepasty = { - gid = 2796546855; #genid bepasty + gid = genid "bepasty"; }; }; diff --git a/krebs/3modules/fetchWallpaper.nix b/krebs/3modules/fetchWallpaper.nix index 83ecf4177..f320c7505 100644 --- a/krebs/3modules/fetchWallpaper.nix +++ b/krebs/3modules/fetchWallpaper.nix @@ -51,7 +51,7 @@ let imp = { users.users.fetchWallpaper = { name = "fetchWallpaper"; - uid = 3332383611; #genid fetchWallpaper + uid = genid "fetchWallpaper"; description = "fetchWallpaper user"; home = cfg.stateDir; createHome = true; diff --git a/krebs/3modules/git.nix b/krebs/3modules/git.nix index 234129497..e6267d7e6 100644 --- a/krebs/3modules/git.nix +++ b/krebs/3modules/git.nix @@ -145,14 +145,14 @@ let ]) (filter (x: hasAttr "allow-receive-ref" x.perm) cfg.rules)); }; - users.extraUsers = singleton { + users.extraUsers = singleton rec { description = "Git repository hosting user"; name = "git"; shell = "/bin/sh"; openssh.authorizedKeys.keys = mapAttrsToList (_: makeAuthorizedKey git-ssh-command) config.krebs.users; - uid = 129318403; # genid git + uid = genid name; }; }; @@ -238,9 +238,9 @@ let }; }; - fcgitwrap-user = { + fcgitwrap-user = rec { name = "fcgiwrap"; - uid = 2867890860; # genid fcgiwrap + uid = genid name; group = "fcgiwrap"; }; diff --git a/krebs/3modules/github-hosts-sync.nix b/krebs/3modules/github-hosts-sync.nix index 5503ee8d6..2aa18d53a 100644 --- a/krebs/3modules/github-hosts-sync.nix +++ b/krebs/3modules/github-hosts-sync.nix @@ -56,9 +56,9 @@ let }; }; - user = { + user = rec { name = "github-hosts-sync"; - uid = 3220554646; # genid github-hosts-sync + uid = genid name; }; # TODO move to lib? diff --git a/krebs/3modules/go.nix b/krebs/3modules/go.nix index 793d1f60d..08a93dab7 100644 --- a/krebs/3modules/go.nix +++ b/krebs/3modules/go.nix @@ -1,6 +1,5 @@ { config, lib, pkgs, ... }: -with builtins; with lib; let @@ -31,9 +30,9 @@ let bind = mkDefault "127.0.0.1"; }; - users.extraUsers.go = { + users.extraUsers.go = rec { name = "go"; - uid = 42774411; #genid go + uid = genid name; description = "go url shortener user"; home = "/var/lib/go"; createHome = true; diff --git a/krebs/3modules/realwallpaper.nix b/krebs/3modules/realwallpaper.nix index 7e02538f5..b377368f7 100644 --- a/krebs/3modules/realwallpaper.nix +++ b/krebs/3modules/realwallpaper.nix @@ -1,13 +1,7 @@ arg@{ config, lib, pkgs, ... }: +with lib; let - inherit (lib) - mkEnableOption - mkOption - types - mkIf - ; - cfg = config.krebs.realwallpaper; out = { @@ -89,7 +83,7 @@ let }; users.extraUsers.realwallpaper = { - uid = 2009435407; #genid realwallpaper + uid = genid "realwallpaper"; home = cfg.workingDir; createHome = true; }; diff --git a/krebs/3modules/retiolum.nix b/krebs/3modules/retiolum.nix index 28ac67306..e0e2692a8 100644 --- a/krebs/3modules/retiolum.nix +++ b/krebs/3modules/retiolum.nix @@ -133,9 +133,9 @@ let }; }; - user = { + user = rec { name = "retiolum"; - uid = 301281149; # genid retiolum + uid = genid name; }; tinc = cfg.tincPackage; diff --git a/krebs/3modules/tinc_graphs.nix b/krebs/3modules/tinc_graphs.nix index ba81dd416..1f32c2e59 100644 --- a/krebs/3modules/tinc_graphs.nix +++ b/krebs/3modules/tinc_graphs.nix @@ -120,7 +120,7 @@ let }; users.extraUsers.tinc_graphs = { - uid = 3925439960; #genid tinc_graphs + uid = genid "tinc_graphs"; home = "/var/spool/tinc_graphs"; }; diff --git a/krebs/3modules/urlwatch.nix b/krebs/3modules/urlwatch.nix index 206bc5697..31cbfcf6e 100644 --- a/krebs/3modules/urlwatch.nix +++ b/krebs/3modules/urlwatch.nix @@ -5,7 +5,6 @@ # cache = url: "${cfg.dataDir}/.urlwatch/cache/${hashString "sha1" url}" # TODO hooks.py -with builtins; with lib; let cfg = config.krebs.urlwatch; @@ -136,9 +135,9 @@ let }; }; - user = { + user = rec { name = "urlwatch"; - uid = 3467631196; # genid urlwatch + uid = genid name; }; in out diff --git a/krebs/4lib/default.nix b/krebs/4lib/default.nix index 1cabeae27..dfc51bbe4 100644 --- a/krebs/4lib/default.nix +++ b/krebs/4lib/default.nix @@ -7,6 +7,8 @@ let out = rec { eq = x: y: x == y; + mod = x: y: x - y * (x / y); + addName = name: set: set // { inherit name; }; @@ -17,6 +19,7 @@ let out = rec { dir.has-default-nix = path: pathExists (path + "/default.nix"); dns = import ./dns.nix { inherit lib; }; + genid = import ./genid.nix { lib = lib // out; }; git = import ./git.nix { lib = lib // out; }; listset = import ./listset.nix { inherit lib; }; shell = import ./shell.nix { inherit lib; }; diff --git a/krebs/4lib/genid.nix b/krebs/4lib/genid.nix new file mode 100644 index 000000000..0aed1d351 --- /dev/null +++ b/krebs/4lib/genid.nix @@ -0,0 +1,37 @@ +{ lib, ... }: +with lib; +with builtins; +let out = genid; + + # id = genid s = (hash s + min) % max + # min <= genid s < max + # + # min = 2^24 = 16777216 = 0x001000000 + # max = 2^32 = 4294967296 = 0x100000000 + # + # id is bigger than UID of nobody and GID of nogroup + # see and some spare for stuff like lxd. + # + # :: str -> uint32 + genid = s: sum16 (addmod16_16777216 (hash s)); + + # :: str -> list8 uint4 + hash = s: + map hexint (stringToCharacters (substring 32 8 (hashString "sha1" s))); + + # :: list uint -> uint + sum16 = foldl (a: i: a * 16 + i) 0; + + # :: list8 uint4 -> list1 uint8 ++ list6 uint4 + addmod16_16777216 = x: let + a = 16 * head x + head (tail x); + d = tail (tail x); + in [(mod (a + 1) 256)] ++ d; + + # :: char -> uint4 + hexint = x: hexvals.${toLower x}; + + # :: attrset char uint4 + hexvals = listToAttrs (imap (i: c: { name = c; value = i - 1; }) + (stringToCharacters "0123456789abcdef")); +in out diff --git a/krebs/5pkgs/genid/default.nix b/krebs/5pkgs/genid/default.nix deleted file mode 100644 index c75bec317..000000000 --- a/krebs/5pkgs/genid/default.nix +++ /dev/null @@ -1,22 +0,0 @@ -{ lib, pkgs, ... }: - -pkgs.writeScriptBin "genid" '' - #! /bin/sh - # usage: genid NAME - set -euf - - export PATH=${lib.makeSearchPath "bin" (with pkgs; [ - bc - coreutils - ])} - - name=$1 - hash=$(printf %s "$name" | sha1sum | cut -d\ -f1 | tr a-f A-F) - echo " - min=2^24 # bigger than nobody and nogroup, see - # and some spare for stuff like lxd. - max=2^32 # see 2^(8*sizeof(uid_t)) - ibase=16 - ($hash + min) % max - " | bc -'' diff --git a/lass/1systems/mors.nix b/lass/1systems/mors.nix index 4ba9df6f9..8af096f51 100644 --- a/lass/1systems/mors.nix +++ b/lass/1systems/mors.nix @@ -184,7 +184,6 @@ cac sshpass get - genid teamspeak_client hashPassword ]; diff --git a/lass/2configs/libvirt.nix b/lass/2configs/libvirt.nix index 368722e77..7520a0e36 100644 --- a/lass/2configs/libvirt.nix +++ b/lass/2configs/libvirt.nix @@ -1,4 +1,4 @@ -{ config, pkgs, ... }: +{ config, lib, pkgs, ... }: let mainUser = config.users.extraUsers.mainUser; @@ -8,7 +8,7 @@ in { users.extraUsers = { libvirt = { - uid = 358821352; # genid libvirt + uid = lib.genid "libvirt"; description = "user for running libvirt stuff"; home = "/home/libvirt"; useDefaultShell = true; diff --git a/lass/2configs/skype.nix b/lass/2configs/skype.nix index 6a226441b..d62a18a52 100644 --- a/lass/2configs/skype.nix +++ b/lass/2configs/skype.nix @@ -1,4 +1,4 @@ -{ config, pkgs, ... }: +{ config, lib, pkgs, ... }: let mainUser = config.users.extraUsers.mainUser; @@ -7,7 +7,7 @@ in { users.extraUsers = { skype = { name = "skype"; - uid = 2259819492; #genid skype + uid = lib.genid "skype"; description = "user for running skype"; home = "/home/skype"; useDefaultShell = true; diff --git a/lass/2configs/weechat.nix b/lass/2configs/weechat.nix index 18007ed61..6a257f0bb 100644 --- a/lass/2configs/weechat.nix +++ b/lass/2configs/weechat.nix @@ -8,7 +8,7 @@ users.extraUsers.chat = { home = "/home/chat"; - uid = 986764891; # genid chat + uid = lib.genid "chat"; useDefaultShell = true; createHome = true; openssh.authorizedKeys.keys = [ diff --git a/lass/3modules/newsbot-js.nix b/lass/3modules/newsbot-js.nix index 6d87d256d..5e340b26f 100644 --- a/lass/3modules/newsbot-js.nix +++ b/lass/3modules/newsbot-js.nix @@ -51,7 +51,7 @@ let imp = { users.extraUsers.newsbot-js = { name = "newsbot-js"; - uid = 1616759810; #genid newsbot-js + uid = genid "newsbot-js"; description = "newsbot-js user"; home = "/var/empty"; }; diff --git a/lass/3modules/owncloud_nginx.nix b/lass/3modules/owncloud_nginx.nix index a0db87b0b..0cb11846c 100644 --- a/lass/3modules/owncloud_nginx.nix +++ b/lass/3modules/owncloud_nginx.nix @@ -207,7 +207,7 @@ let # }; #}); users.users.nobody_oc = { - uid = 1651469147; # genid nobody_oc + uid = genid "nobody_oc"; useDefaultShell = true; }; }; diff --git a/lass/3modules/wordpress_nginx.nix b/lass/3modules/wordpress_nginx.nix index 2f31f6e02..974aacd83 100644 --- a/lass/3modules/wordpress_nginx.nix +++ b/lass/3modules/wordpress_nginx.nix @@ -229,7 +229,7 @@ let }; }); users.users.nobody2 = mkDefault { - uid = mkDefault 125816384; # genid nobody2 + uid = mkDefault (genid "nobody2"); useDefaultShell = mkDefault true; }; }; diff --git a/makefu/1systems/vbob.nix b/makefu/1systems/vbob.nix index a24cefd0d..b8c02cb67 100644 --- a/makefu/1systems/vbob.nix +++ b/makefu/1systems/vbob.nix @@ -59,7 +59,6 @@ in { buildbot buildbot-slave get - genid ]; networking.firewall.allowedTCPPorts = [ diff --git a/makefu/3modules/buildbot/master.nix b/makefu/3modules/buildbot/master.nix index 58e2f8175..09edac94d 100644 --- a/makefu/3modules/buildbot/master.nix +++ b/makefu/3modules/buildbot/master.nix @@ -204,7 +204,7 @@ let imp = { users.extraUsers.buildbotMaster = { - uid = 672626386; #genid buildbotMaster + uid = genid "buildbotMaster"; description = "Buildbot Master"; home = cfg.workDir; createHome = false; diff --git a/makefu/3modules/buildbot/slave.nix b/makefu/3modules/buildbot/slave.nix index 69d0361bf..7c9ea79c0 100644 --- a/makefu/3modules/buildbot/slave.nix +++ b/makefu/3modules/buildbot/slave.nix @@ -127,7 +127,7 @@ let imp = { users.extraUsers.buildbotSlave = { - uid = 1408105834; #genid buildbotMaster + uid = genid "buildbotSlave"; description = "Buildbot Slave"; home = cfg.workDir; createHome = false; diff --git a/tv/1systems/wu.nix b/tv/1systems/wu.nix index a768e781b..54ceb7783 100644 --- a/tv/1systems/wu.nix +++ b/tv/1systems/wu.nix @@ -16,7 +16,6 @@ with lib; environment.systemPackages = with pkgs; [ # stockholm - genid gnumake hashPassword lentil diff --git a/tv/1systems/xu.nix b/tv/1systems/xu.nix index eac36fafb..1f3e010a4 100644 --- a/tv/1systems/xu.nix +++ b/tv/1systems/xu.nix @@ -19,7 +19,6 @@ with lib; environment.systemPackages = with pkgs; [ # stockholm - genid gnumake hashPassword lentil diff --git a/tv/2configs/charybdis.nix b/tv/2configs/charybdis.nix index 80c6f7c4a..f9ab3da68 100644 --- a/tv/2configs/charybdis.nix +++ b/tv/2configs/charybdis.nix @@ -72,9 +72,9 @@ let }; }; - user = { + user = rec { name = "charybdis"; - uid = 3748224544; # genid charybdis + uid = genid name; }; configFile = toFile "charybdis-ircd.conf" '' diff --git a/tv/2configs/pulse.nix b/tv/2configs/pulse.nix index 0ddc52789..3db3532d5 100644 --- a/tv/2configs/pulse.nix +++ b/tv/2configs/pulse.nix @@ -69,12 +69,10 @@ in }; }; - users = let - id = 3768151709; # genid pulse - in { - groups.pulse.gid = id; + users = { + groups.pulse.gid = config.users.users.pulse.uid; users.pulse = { - uid = id; + uid = genid "pulse"; group = "pulse"; extraGroups = [ "audio" ]; home = "${runDir}/home"; diff --git a/tv/3modules/consul.nix b/tv/3modules/consul.nix index ccdee07f5..5c955fdb5 100644 --- a/tv/3modules/consul.nix +++ b/tv/3modules/consul.nix @@ -109,9 +109,9 @@ let }; }; - user = { + user = rec { name = "consul"; - uid = 2999951406; # genid consul + uid = genid name; }; in diff --git a/tv/3modules/ejabberd.nix b/tv/3modules/ejabberd.nix index 6b231fb56..581e10074 100644 --- a/tv/3modules/ejabberd.nix +++ b/tv/3modules/ejabberd.nix @@ -53,9 +53,9 @@ let }; }; - user = { + user = rec { name = "ejabberd"; - uid = 3499746127; # genid ejabberd + uid = genid name; }; my-ejabberdctl = pkgs.writeScriptBin "ejabberdctl" '' -- cgit v1.3.1 From 669e4be273ac2abe9505ca6411d5ee37f1771d4c Mon Sep 17 00:00:00 2001 From: makefu Date: Sat, 26 Dec 2015 11:06:11 +0100 Subject: k 5 Reaktor/plugins: converted plugins from makefu/2/Reaktor --- krebs/5pkgs/Reaktor/plugins.nix | 124 ++++++++++++++++++++++----- krebs/5pkgs/Reaktor/scripts/random-issue.sh | 20 +++++ krebs/5pkgs/Reaktor/scripts/sed-plugin.py | 53 ++++++++++++ krebs/5pkgs/Reaktor/scripts/shack-correct.sh | 6 ++ makefu/2configs/Reaktor/full.nix | 18 ---- makefu/2configs/Reaktor/random-emoji.nix | 26 ------ makefu/2configs/Reaktor/random-emoji.sh | 6 -- makefu/2configs/Reaktor/random-issue.sh | 20 ----- makefu/2configs/Reaktor/sed-plugin.nix | 18 ---- makefu/2configs/Reaktor/sed-plugin.py | 53 ------------ makefu/2configs/Reaktor/shack-correct.nix | 20 ----- makefu/2configs/Reaktor/shack-correct.sh | 6 -- makefu/2configs/Reaktor/simpleExtend.nix | 19 ---- makefu/2configs/Reaktor/stockholmLentil.nix | 27 ------ makefu/2configs/Reaktor/titlebot.nix | 38 -------- 15 files changed, 181 insertions(+), 273 deletions(-) create mode 100644 krebs/5pkgs/Reaktor/scripts/random-issue.sh create mode 100644 krebs/5pkgs/Reaktor/scripts/sed-plugin.py create mode 100644 krebs/5pkgs/Reaktor/scripts/shack-correct.sh delete mode 100644 makefu/2configs/Reaktor/full.nix delete mode 100644 makefu/2configs/Reaktor/random-emoji.nix delete mode 100644 makefu/2configs/Reaktor/random-emoji.sh delete mode 100644 makefu/2configs/Reaktor/random-issue.sh delete mode 100644 makefu/2configs/Reaktor/sed-plugin.nix delete mode 100644 makefu/2configs/Reaktor/sed-plugin.py delete mode 100644 makefu/2configs/Reaktor/shack-correct.nix delete mode 100644 makefu/2configs/Reaktor/shack-correct.sh delete mode 100644 makefu/2configs/Reaktor/simpleExtend.nix delete mode 100644 makefu/2configs/Reaktor/stockholmLentil.nix delete mode 100644 makefu/2configs/Reaktor/titlebot.nix (limited to 'krebs/5pkgs') diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix index 05ede38e1..3b2508862 100644 --- a/krebs/5pkgs/Reaktor/plugins.nix +++ b/krebs/5pkgs/Reaktor/plugins.nix @@ -1,38 +1,118 @@ { stdenv, lib, pkgs, makeWrapper }: rec { - buildReaktorPlugin = { name - # TODO: profiles - , extraConfig + # Begin API + buildBaseReaktorPlugin = { name + , config # python extra configuration for plugin , phases ? [] , ... } @ attrs: stdenv.mkDerivation (attrs // { name = "Reaktor-plugin-" + name; - phases = phases ++ [ "installPhase" ]; isReaktorPlugin = true; }); - random-emoji = buildReaktorPlugin rec { - name = "random-emoji"; - src = ./scripts/random-emoji.sh; + buildSimpleReaktorPlugin = name: { script + , path ? [] + , env ? {} + , pattern ? "" + , ... } @ attrs: + let + path_env = { "PATH" = lib.makeSearchPath "bin" (path ++ [ pkgs.coreutils ]); }; + src_dir = pkgs.substituteAll ( { + inherit name; + dir = "bin"; + isExecutable = true; + src = script; + }); + src_file = "${src_dir}/bin/${name}"; + config = '' + public_commands.insert(0,{ + 'capname' : "${name}", + 'pattern' : ${if pattern == "" then + ''indirect_pattern.format("${name}")'' else + ''"${pattern}"'' }, + 'argv' : ["${src_file}"], + 'env' : ${builtins.toJSON path_env // env})}) + ''; + config_file = pkgs.writeText "plugin.py" config; + in buildBaseReaktorPlugin (attrs // rec { + inherit name config; + + phases = [ "installPhase" ]; + buildInputs = [ makeWrapper ]; + installPhase = '' + mkdir -p $out/bin $out/etc/Reaktor + ln -s ${src_file} $out/bin + wrapProgram $out/bin/${name} \ + --prefix PATH : ${path_env.PATH} + ln -s ${config_file} $out/etc/Reaktor/plugin.py + ''; + + }); + # End API + + # Begin Plugins + random-emoji = buildSimpleReaktorPlugin "emoji" { + path = with pkgs; [ gnused gnugrep xmlstarlet curl ]; + script = ./scripts/random-emoji.sh; + }; + + sed-plugin = buildSimpleReaktorPlugin "sed-plugin" { + path = [ pkgs.gnused ]; + # only support s///gi the plugin needs to see every msg + # TODO: this will eat up the last regex, fix Reaktor to support fallthru + pattern = "^(?P.*)$$"; + script = ./scripts/sed-plugin.py; + }; + + shack-correct = buildSimpleReaktorPlugin "shack-correct" { + path = [ pkgs.gnused ]; + pattern = "^(?P.*Shack.*)$$"; + script = ./scripts/shack-correct.sh; + }; + + nixos-version = buildSimpleReaktorPlugin "nixos-version" { + script = pkgs.writeScript "nixos-version" '' + #! /bin/sh + . /etc/os-release + echo "$PRETTY_NAME" + ''; + }; + stockholm-issue = buildSimpleReaktorPlugin "stockholm-issue" { + script = ./scripts/random-issue.sh; + path = with pkgs; [ git gnused lentil ]; + env = { "origin"= "http://cgit.gum/stockholm"; }; + }; + + titlebot = + let + pypkgs = pkgs.python3Packages; + titlebot_cmds = pypkgs.buildPythonPackage { + name = "titlebot_cmds"; + propagatedBuildInputs = with pypkgs; [ setuptools ]; + src = pkgs.fetchurl { + url = "https://github.com/makefu/reaktor-titlebot/archive/2.1.0.tar.gz"; + sha256 = "0wvf09wmk8b52f9j65qrw81nwrhs9pfhijwrlkzp5l7l2q8cjkp6"; + }; + }; + in buildBaseReaktorPlugin rec { + name = "titlebot"; phases = [ "installPhase" ]; - buildInputs = [ makeWrapper ]; installPhase = '' - mkdir -p $out/bin - install -vm 755 ${src} $out/bin/random-emoji.sh - wrapProgram $out/bin/random-emoji.sh \ - --prefix PATH : ${lib.makeSearchPath "bin" (with pkgs; [ - coreutils - gnused - gnugrep - xmlstarlet - curl])}; + mkdir -p $out + ln -s ${titlebot_cmds}/* $out ''; - extraConfig = '' - public_commands.insert(0,{ - 'capname' : "emoji", - 'pattern' : indirect_pattern.format("emoji"), - 'argv' : ["random-emoji.sh"]) + config = '' + def titlebot_cmd(cmd): + from os import environ + return { 'capname': cmd, + 'env': { 'TITLEDB': + environ['state_dir']+'/suggestions.json' }, + 'pattern': '^\\.' + cmd + '\\s*(?:\\s+(?P.*))?$$', + 'argv': [ '${titlebot_cmds}/bin/' + cmd ] } + for i in ['up','help','list','top','new']: + public_commands.insert(0,titlebot_cmd(i)) + commands.insert(0,titlebot_cmd('clear')) ''; }; } diff --git a/krebs/5pkgs/Reaktor/scripts/random-issue.sh b/krebs/5pkgs/Reaktor/scripts/random-issue.sh new file mode 100644 index 000000000..5c47c6156 --- /dev/null +++ b/krebs/5pkgs/Reaktor/scripts/random-issue.sh @@ -0,0 +1,20 @@ +#! /bin/sh +set -eu +# requires env: +# $state_dir +# $origin + +# in PATH: git,lentil,coreutils +subdir=`echo "$1" | tr -dc "[:alnum:]"` +name=`echo "$origin" | tr -dc "[:alnum:]"` +track="$state_dir/$name-checkout" +(if test -e "$track" ;then + cd "$track" + git fetch origin master + git reset --hard origin/master +else + git clone "$origin" "$track" +fi) >&2 + +cd "$track" +lentil "${subdir:-.}" -f csv | sed 1d | shuf | head -1 diff --git a/krebs/5pkgs/Reaktor/scripts/sed-plugin.py b/krebs/5pkgs/Reaktor/scripts/sed-plugin.py new file mode 100644 index 000000000..8103c9585 --- /dev/null +++ b/krebs/5pkgs/Reaktor/scripts/sed-plugin.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +# Usage: +# _from=krebs state_dir=. python sed-plugin.py 'dick butt' +# _from=krebs state_dir=. python sed-plugin.py 's/t/l/g' +## dick bull +import shelve +from os import environ +from os.path import join +from sys import argv +d = shelve.open(join(environ['state_dir'],'sed-plugin.shelve'),writeback=True) +usr = environ['_from'] +import re + +def is_regex(line): + myre = re.compile(r'^s/((?:\\/|[^/])+)/((?:\\/|[^/])*)/([ig]*)$') + return myre.match(line) + +line = argv[1] +m = is_regex(line) + +if m: + f,t,flagstr = m.groups() + fn = f.replace('\/','/') + tn = t.replace('\/','/') + flags = 0 + count = 1 + if flagstr: + if 'i' in flagstr: + flags = re.IGNORECASE + if 'g' in flagstr: + count = 0 + else: + flagstr = '' + last = d.get(usr,None) + if last: + #print(re.sub(fn,tn,last,count=count,flags=flags)) + from subprocess import Popen,PIPE + p = Popen(['sed','s/{}/{}/{}'.format(f,t,flagstr)],stdin=PIPE,stdout=PIPE ) + so,se = p.communicate(bytes("{}\n".format(last),"UTF-8")) + if p.returncode: + print("something went wrong when trying to process your regex: {}".format(se.decode())) + ret = so.decode() + print("\x1b[1m{}\x1b[0m meinte: {}".format(usr,ret.strip())) + if ret: + d[usr] = ret + + else: + print("no last message") +else: + d[usr] = line + +d.close() diff --git a/krebs/5pkgs/Reaktor/scripts/shack-correct.sh b/krebs/5pkgs/Reaktor/scripts/shack-correct.sh new file mode 100644 index 000000000..3b4d04f80 --- /dev/null +++ b/krebs/5pkgs/Reaktor/scripts/shack-correct.sh @@ -0,0 +1,6 @@ +#! /bin/sh +set -eu +printf "Sie meinten wohl \"" +echo -n $@ | sed 's/Shack/shack/g' +echo "\"" +echo "${_from}--" diff --git a/makefu/2configs/Reaktor/full.nix b/makefu/2configs/Reaktor/full.nix deleted file mode 100644 index 50620890f..000000000 --- a/makefu/2configs/Reaktor/full.nix +++ /dev/null @@ -1,18 +0,0 @@ -_: -{ - # implementation of the complete Reaktor bot - imports = [ - #./stockholmLentil.nix - ./simpleExtend.nix - ./random-emoji.nix - ./titlebot.nix - ./shack-correct.nix - ./sed-plugin.nix - ]; - krebs.Reaktor.nickname = "Reaktor|bot"; - krebs.Reaktor.enable = true; - - krebs.Reaktor.extraEnviron = { - REAKTOR_CHANNELS = "#krebs,#binaergewitter,#shackspace"; - }; -} diff --git a/makefu/2configs/Reaktor/random-emoji.nix b/makefu/2configs/Reaktor/random-emoji.nix deleted file mode 100644 index 3113a826b..000000000 --- a/makefu/2configs/Reaktor/random-emoji.nix +++ /dev/null @@ -1,26 +0,0 @@ -{ config, lib, pkgs, ... }: - -with pkgs; -let - rpkg = pkgs.substituteAll( { - name="random-emoji"; - dir= "bin"; - isExecutable=true; - src= ./random-emoji.sh; - }); - rpkg-path = lib.makeSearchPath "bin" (with pkgs; [ - coreutils - gnused - gnugrep - xmlstarlet - curl]); -in { - # TODO: make origin a variable, <- module is generic enough to handle different origins, not only stockholm - krebs.Reaktor.extraConfig = '' - public_commands.insert(0,{ - 'capname' : "emoji", - 'pattern' : indirect_pattern.format("emoji"), - 'argv' : ["${rpkg}/bin/random-emoji"], - 'env' : { 'PATH':'${rpkg-path}' } }) - ''; -} diff --git a/makefu/2configs/Reaktor/random-emoji.sh b/makefu/2configs/Reaktor/random-emoji.sh deleted file mode 100644 index 386aa68b9..000000000 --- a/makefu/2configs/Reaktor/random-emoji.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -curl http://emojicons.com/random -s | \ - grep data-text | \ - sed -n 's/.*>\(.*\)<\/textarea>/\1/p' | \ - head -n 1 | \ - xmlstarlet unesc diff --git a/makefu/2configs/Reaktor/random-issue.sh b/makefu/2configs/Reaktor/random-issue.sh deleted file mode 100644 index 5c47c6156..000000000 --- a/makefu/2configs/Reaktor/random-issue.sh +++ /dev/null @@ -1,20 +0,0 @@ -#! /bin/sh -set -eu -# requires env: -# $state_dir -# $origin - -# in PATH: git,lentil,coreutils -subdir=`echo "$1" | tr -dc "[:alnum:]"` -name=`echo "$origin" | tr -dc "[:alnum:]"` -track="$state_dir/$name-checkout" -(if test -e "$track" ;then - cd "$track" - git fetch origin master - git reset --hard origin/master -else - git clone "$origin" "$track" -fi) >&2 - -cd "$track" -lentil "${subdir:-.}" -f csv | sed 1d | shuf | head -1 diff --git a/makefu/2configs/Reaktor/sed-plugin.nix b/makefu/2configs/Reaktor/sed-plugin.nix deleted file mode 100644 index a451e0d3e..000000000 --- a/makefu/2configs/Reaktor/sed-plugin.nix +++ /dev/null @@ -1,18 +0,0 @@ -{ config, lib, pkgs, ... }: - -with pkgs; -let - script = ./sed-plugin.py; -in { - #TODO: this will eat up the last regex, fix Reaktor - krebs.Reaktor.extraConfig = '' - public_commands.append({ - 'capname' : "sed-plugin", - # only support s///gi - 'pattern' : '^(?P.*)$$', - 'argv' : ["${pkgs.python3}/bin/python3","${script}"], - 'env' : { 'state_dir' : workdir, - 'PATH':'${lib.makeSearchPath "bin" [pkgs.gnused]}' }}) - ''; -} - diff --git a/makefu/2configs/Reaktor/sed-plugin.py b/makefu/2configs/Reaktor/sed-plugin.py deleted file mode 100644 index 8103c9585..000000000 --- a/makefu/2configs/Reaktor/sed-plugin.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 - -# Usage: -# _from=krebs state_dir=. python sed-plugin.py 'dick butt' -# _from=krebs state_dir=. python sed-plugin.py 's/t/l/g' -## dick bull -import shelve -from os import environ -from os.path import join -from sys import argv -d = shelve.open(join(environ['state_dir'],'sed-plugin.shelve'),writeback=True) -usr = environ['_from'] -import re - -def is_regex(line): - myre = re.compile(r'^s/((?:\\/|[^/])+)/((?:\\/|[^/])*)/([ig]*)$') - return myre.match(line) - -line = argv[1] -m = is_regex(line) - -if m: - f,t,flagstr = m.groups() - fn = f.replace('\/','/') - tn = t.replace('\/','/') - flags = 0 - count = 1 - if flagstr: - if 'i' in flagstr: - flags = re.IGNORECASE - if 'g' in flagstr: - count = 0 - else: - flagstr = '' - last = d.get(usr,None) - if last: - #print(re.sub(fn,tn,last,count=count,flags=flags)) - from subprocess import Popen,PIPE - p = Popen(['sed','s/{}/{}/{}'.format(f,t,flagstr)],stdin=PIPE,stdout=PIPE ) - so,se = p.communicate(bytes("{}\n".format(last),"UTF-8")) - if p.returncode: - print("something went wrong when trying to process your regex: {}".format(se.decode())) - ret = so.decode() - print("\x1b[1m{}\x1b[0m meinte: {}".format(usr,ret.strip())) - if ret: - d[usr] = ret - - else: - print("no last message") -else: - d[usr] = line - -d.close() diff --git a/makefu/2configs/Reaktor/shack-correct.nix b/makefu/2configs/Reaktor/shack-correct.nix deleted file mode 100644 index 8f30807f1..000000000 --- a/makefu/2configs/Reaktor/shack-correct.nix +++ /dev/null @@ -1,20 +0,0 @@ -{ config, lib, pkgs, ... }: - -with pkgs; -let - script = pkgs.substituteAll ( { - name="shack-correct"; - isExecutable=true; - dir = ""; - src = ./shack-correct.sh; - }); -in { - krebs.Reaktor.extraConfig = '' - public_commands.insert(0,{ - 'capname' : "shack-correct", - 'pattern' : '^(?P.*Shack.*)$$', - 'argv' : ["${script}"], - 'env' : { }}) - ''; -} - diff --git a/makefu/2configs/Reaktor/shack-correct.sh b/makefu/2configs/Reaktor/shack-correct.sh deleted file mode 100644 index 3b4d04f80..000000000 --- a/makefu/2configs/Reaktor/shack-correct.sh +++ /dev/null @@ -1,6 +0,0 @@ -#! /bin/sh -set -eu -printf "Sie meinten wohl \"" -echo -n $@ | sed 's/Shack/shack/g' -echo "\"" -echo "${_from}--" diff --git a/makefu/2configs/Reaktor/simpleExtend.nix b/makefu/2configs/Reaktor/simpleExtend.nix deleted file mode 100644 index 95175a4e0..000000000 --- a/makefu/2configs/Reaktor/simpleExtend.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ config, lib, pkgs, ... }: - -with pkgs; -let - nixos-version-script = pkgs.writeScript "nix-version" '' - #! /bin/sh - . /etc/os-release - echo "$PRETTY_NAME" - ''; -in { - krebs.Reaktor.extraConfig = '' - public_commands.insert(0,{ - 'capname' : "nixos-version", - 'pattern' : indirect_pattern.format("nixos-version"), - 'argv' : ["${nixos-version-script}"], - 'env' : { 'state_dir': workdir } }) - ''; -} - diff --git a/makefu/2configs/Reaktor/stockholmLentil.nix b/makefu/2configs/Reaktor/stockholmLentil.nix deleted file mode 100644 index 21f0305fb..000000000 --- a/makefu/2configs/Reaktor/stockholmLentil.nix +++ /dev/null @@ -1,27 +0,0 @@ -{ config, lib, pkgs, ... }: - -with pkgs; -let - random-issue = pkgs.substituteAll( { - name="random-issue"; - dir= "bin"; - isExecutable=true; - src= ./random-issue.sh; - }); - random-issue-path = lib.makeSearchPath "bin" (with pkgs; [ - coreutils - git - gnused - lentil]); -in { - # TODO: make origin a variable, <- module is generic enough to handle different origins, not only stockholm - krebs.Reaktor.extraConfig = '' - public_commands.insert(0,{ - 'capname' : "stockholm-issue", - 'pattern' : indirect_pattern.format("stockholm-issue"), - 'argv' : ["${random-issue}/bin/random-issue"], - 'env' : { 'state_dir': workdir, - 'PATH':'${random-issue-path}', - 'origin':'http://cgit.pnp/stockholm' } }) - ''; -} diff --git a/makefu/2configs/Reaktor/titlebot.nix b/makefu/2configs/Reaktor/titlebot.nix deleted file mode 100644 index 9ef02548b..000000000 --- a/makefu/2configs/Reaktor/titlebot.nix +++ /dev/null @@ -1,38 +0,0 @@ -{ stdenv,config, lib, pkgs, ... }: - -with pkgs; -let - pypkgs = pkgs.python3Packages; - titlebot_cmds = pypkgs.buildPythonPackage { - name = "titlebot_cmds"; - propagatedBuildInputs = with pypkgs; [ setuptools ]; - src = fetchurl { - # https://github.com/makefu/reaktor-titlebot tag 2.1.0 - url = "https://github.com/makefu/reaktor-titlebot/archive/2.1.0.tar.gz"; - sha256 = "0wvf09wmk8b52f9j65qrw81nwrhs9pfhijwrlkzp5l7l2q8cjkp6"; - }; - }; - pub_cmds = ["up" "help" "list" "top" "highest" "undo" ]; - priv_cmds = [ "clear" ]; -in { - # TODO: write identify file in - # {config.users.extraUsers.Reaktor.home}/state/admin.lst - krebs.Reaktor.extraConfig = '' - def titlebot_cmd(cmd): - return { - 'capname': cmd, - 'env': { - 'TITLEDB': - '${config.users.extraUsers.Reaktor.home}/suggestions.json' - }, - 'pattern': '^\\.' + cmd + '\\s*(?:\\s+(?P.*))?$$', - 'argv': [ '${titlebot_cmds}/bin/' + cmd ] } - # TODO: for each element in ${titlebot_cmds}/bin/* - public_commands.insert(0,titlebot_cmd('up')) - public_commands.insert(0,titlebot_cmd('help')) - public_commands.insert(0,titlebot_cmd('list')) - public_commands.insert(0,titlebot_cmd('top')) - public_commands.insert(0,titlebot_cmd('new')) - commands.insert(0,titlebot_cmd('clear')) - ''; -} -- cgit v1.3.1 From a2f5e7e320bb0fbca0a0694d91e4fb20dc4ef329 Mon Sep 17 00:00:00 2001 From: makefu Date: Sat, 26 Dec 2015 11:31:09 +0100 Subject: k 5 ReaktorPlugins: hotfix for env generation --- krebs/5pkgs/Reaktor/plugins.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'krebs/5pkgs') diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix index 3b2508862..b1a61d3fa 100644 --- a/krebs/5pkgs/Reaktor/plugins.nix +++ b/krebs/5pkgs/Reaktor/plugins.nix @@ -32,7 +32,7 @@ rec { ''indirect_pattern.format("${name}")'' else ''"${pattern}"'' }, 'argv' : ["${src_file}"], - 'env' : ${builtins.toJSON path_env // env})}) + 'env' : ${builtins.toJSON (path_env // env)})}) ''; config_file = pkgs.writeText "plugin.py" config; in buildBaseReaktorPlugin (attrs // rec { -- cgit v1.3.1 From 7bed1761bdbfc3fc7e2df56dcf069511eec2a97d Mon Sep 17 00:00:00 2001 From: makefu Date: Sat, 26 Dec 2015 11:41:41 +0100 Subject: m 3 Reaktor: now supports plugin infra see m/1/pornocauster --- krebs/3modules/Reaktor.nix | 5 ++++- krebs/5pkgs/Reaktor/plugins.nix | 4 ++-- makefu/1systems/pornocauster.nix | 13 +++++++------ 3 files changed, 13 insertions(+), 9 deletions(-) (limited to 'krebs/5pkgs') diff --git a/krebs/3modules/Reaktor.nix b/krebs/3modules/Reaktor.nix index 59058bffc..607eb7cac 100644 --- a/krebs/3modules/Reaktor.nix +++ b/krebs/3modules/Reaktor.nix @@ -9,6 +9,7 @@ let ${cfg.overrideConfig} '' else ""} ## Extra Config + ${concatStringsSep "\n" (map (plug: plug.config) cfg.plugins)} ${cfg.extraConfig} ''; cfg = config.krebs.Reaktor; @@ -35,7 +36,6 @@ let ''; }; - overrideConfig = mkOption { default = null; type = types.nullOr types.str; @@ -44,6 +44,9 @@ let Reaktor default cfg can be retrieved via `reaktor get-config` ''; }; + plugins = mkOption { + default = [pkgs.ReaktorPlugins.nixos-version]; + }; extraConfig = mkOption { default = ""; type = types.string; diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix index b1a61d3fa..5c7b89f5c 100644 --- a/krebs/5pkgs/Reaktor/plugins.nix +++ b/krebs/5pkgs/Reaktor/plugins.nix @@ -32,7 +32,7 @@ rec { ''indirect_pattern.format("${name}")'' else ''"${pattern}"'' }, 'argv' : ["${src_file}"], - 'env' : ${builtins.toJSON (path_env // env)})}) + 'env' : ${builtins.toJSON (path_env // env)} }) ''; config_file = pkgs.writeText "plugin.py" config; in buildBaseReaktorPlugin (attrs // rec { @@ -81,7 +81,7 @@ rec { stockholm-issue = buildSimpleReaktorPlugin "stockholm-issue" { script = ./scripts/random-issue.sh; path = with pkgs; [ git gnused lentil ]; - env = { "origin"= "http://cgit.gum/stockholm"; }; + env = { "origin" = "http://cgit.gum/stockholm"; }; }; titlebot = diff --git a/makefu/1systems/pornocauster.nix b/makefu/1systems/pornocauster.nix index 28b77d330..690e26b36 100644 --- a/makefu/1systems/pornocauster.nix +++ b/makefu/1systems/pornocauster.nix @@ -26,6 +26,7 @@ # services ../2configs/git/brain-retiolum.nix ../2configs/tor.nix + # ../2configs/buildbot-standalone.nix # hardware specifics are in here ../2configs/hw/tp-x220.nix @@ -36,14 +37,14 @@ ]; nixpkgs.config.packageOverrides = pkgs: { tinc = pkgs.tinc_pre; - buildbot = let - pkgs1509 = import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz) {}; - in pkgs1509.buildbot; }; - makefu.buildbot.master.enable = true; - #krebs.Reaktor.enable = true; - #krebs.Reaktor.nickname = "makefu|r"; + krebs.Reaktor = { + enable = true; + nickname = "makefu|r"; + plugins = with pkgs.ReaktorPlugins; [ nixos-version random-emoji ]; + }; + # nix.binaryCaches = [ "http://acng.shack/nixos" "https://cache.nixos.org" ]; environment.systemPackages = with pkgs;[ -- cgit v1.3.1 From 95caa8d7fb6f72cbf5064256f71750096d32a6c0 Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 28 Dec 2015 13:23:34 +0100 Subject: k 5 test/infest-cac-centos7: use defer, loop until we get a working cac box --- krebs/5pkgs/test/infest-cac-centos7/notes | 77 ++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 26 deletions(-) (limited to 'krebs/5pkgs') diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes index cfb074423..3f4fcd859 100755 --- a/krebs/5pkgs/test/infest-cac-centos7/notes +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -8,6 +8,17 @@ set -eufx krebs_cred=${krebs_cred-./cac.json} retiolum_key=${retiolum_key-./retiolum.rsa_key.priv} +clear_defer(){ + echo "${trapstr:-exit}" + trap - INT TERM EXIT KILL +} +defer(){ + if test -z "${debug:-}"; then + trapstr="$1;${trapstr:-exit}" + trap "$trapstr" INT TERM EXIT KILL + fi +} + # Sanity if test ! -r "$krebs_cred";then echo "\$krebs_cred=$krebs_cred must be readable"; exit 1 @@ -24,8 +35,8 @@ export cac_servers_cache=$krebs_secrets/servers_cache.json export cac_tasks_cache=$krebs_secrets/tasks_cache.json export cac_templates_cache=$krebs_secrets/templates_cache.json # we need to receive this key from buildmaster to speed up tinc bootstrap -TRAP="rm -r $krebs_secrets;trap - INT TERM EXIT" -trap "$TRAP" INT TERM EXIT +defer "trap - INT TERM EXIT" +defer "rm -r $krebs_secrets" cat > $sec_file <&1\ - | jq -r .servername) - -id=servername:$name -trap "cac delete $id;$TRAP;exit" INT TERM EXIT -# TODO: timeout? - -wait_login_cac(){ - # timeout - for t in `seq 180`;do - # now we have a working cac server - if cac ssh $1 -o ConnectTimeout=10 \ - cat /etc/redhat-release | \ - grep CentOS ;then - return 0 - fi - sleep 10 - done - return 1 -} -# die on timeout -wait_login_cac $id +# preserve old trap +old_trapstr=$(clear_defer) +while true;do + # Template 26: CentOS7 + # TODO: use cac templates to determine the real Centos7 template in case it changes + name=$( cac build cpu=1 ram=512 storage=10 os=26 2>&1\ + | jq -r .servername) + id=servername:$name + + clear_defer >/dev/null + defer "cac delete $id" + + # TODO: timeout? + + wait_login_cac(){ + # we wait for 15 minutes + for t in `seq 90`;do + # now we have a working cac server + if cac ssh $1 -o ConnectTimeout=10 \ + cat /etc/redhat-release | \ + grep CentOS ;then + return 0 + fi + sleep 10 + done + return 1 + } + # die on timeout + if ! wait_login_cac $id;then + echo "unable to boot a working system within time frame, retrying..." >&2 + echo "Cleaning up old image,last status: $(cac update;cac getserver $id | jq -r .status)" + eval "$(clear_defer)" + else + echo "got a working system" >&2 + fi +done +clear_defer >/dev/null +defer "cac delete $id;$old_trapstr" mkdir -p shared/2configs/temp cac generatenetworking $id > \ -- cgit v1.3.1 From 1798dbaf47fea7793545be2bc78ac5f1c8e27e18 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 30 Dec 2015 04:56:53 +0100 Subject: k 5 test: fix endless loop in test --- krebs/5pkgs/test/infest-cac-centos7/notes | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'krebs/5pkgs') diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes index 3f4fcd859..eee0bfc34 100755 --- a/krebs/5pkgs/test/infest-cac-centos7/notes +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -81,9 +81,10 @@ while true;do if ! wait_login_cac $id;then echo "unable to boot a working system within time frame, retrying..." >&2 echo "Cleaning up old image,last status: $(cac update;cac getserver $id | jq -r .status)" - eval "$(clear_defer)" + eval "$(clear_defer | sed 's/;exit//')" else echo "got a working system" >&2 + break fi done clear_defer >/dev/null -- cgit v1.3.1 From f0ce9a72a6595f521f68a156aa46b2372a391d38 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 30 Dec 2015 11:52:22 +0100 Subject: k 5 Reaktor.plugins: fix sed-plugin --- krebs/5pkgs/Reaktor/plugins.nix | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'krebs/5pkgs') diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix index 5c7b89f5c..7490be4ca 100644 --- a/krebs/5pkgs/Reaktor/plugins.nix +++ b/krebs/5pkgs/Reaktor/plugins.nix @@ -14,6 +14,7 @@ rec { buildSimpleReaktorPlugin = name: { script , path ? [] , env ? {} + , append_rule ? false # append the rule instead of insert , pattern ? "" , ... } @ attrs: let @@ -26,7 +27,7 @@ rec { }); src_file = "${src_dir}/bin/${name}"; config = '' - public_commands.insert(0,{ + public_commands.${if append_rule then "append(" else "insert(0," }{ 'capname' : "${name}", 'pattern' : ${if pattern == "" then ''indirect_pattern.format("${name}")'' else @@ -58,9 +59,10 @@ rec { }; sed-plugin = buildSimpleReaktorPlugin "sed-plugin" { - path = [ pkgs.gnused ]; + path = [ pkgs.gnused pkgs.python3 ]; # only support s///gi the plugin needs to see every msg # TODO: this will eat up the last regex, fix Reaktor to support fallthru + append_rule = true; pattern = "^(?P.*)$$"; script = ./scripts/sed-plugin.py; }; @@ -105,7 +107,7 @@ rec { config = '' def titlebot_cmd(cmd): from os import environ - return { 'capname': cmd, + return { 'capname': None, 'env': { 'TITLEDB': environ['state_dir']+'/suggestions.json' }, 'pattern': '^\\.' + cmd + '\\s*(?:\\s+(?P.*))?$$', -- cgit v1.3.1 From f0e802d2593ebe7101968deb3593f1c120f552fd Mon Sep 17 00:00:00 2001 From: makefu Date: Sat, 2 Jan 2016 21:36:51 +0100 Subject: k 5 test/infest-cac-centos7: add timeouts, error handling --- krebs/5pkgs/test/infest-cac-centos7/notes | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) (limited to 'krebs/5pkgs') diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes index eee0bfc34..6bfb6906e 100755 --- a/krebs/5pkgs/test/infest-cac-centos7/notes +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -55,9 +55,16 @@ old_trapstr=$(clear_defer) while true;do # Template 26: CentOS7 # TODO: use cac templates to determine the real Centos7 template in case it changes - name=$( cac build cpu=1 ram=512 storage=10 os=26 2>&1\ - | jq -r .servername) - id=servername:$name + out=$(cac build cpu=1 ram=512 storage=10 os=26 2>&1) + if name=$(echo "$out" | jq -r .servername);then + id=servername:$name + echo "got a working machine, id=$id" + else + echo "Unable to build a virtual machine, retrying in 15 seconds" >&2 + echo "Output of build program: $out" >&2 + sleep 15 + continue + fi clear_defer >/dev/null defer "cac delete $id" @@ -65,8 +72,8 @@ while true;do # TODO: timeout? wait_login_cac(){ - # we wait for 15 minutes - for t in `seq 90`;do + # we wait for 30 minutes + for t in `seq 180`;do # now we have a working cac server if cac ssh $1 -o ConnectTimeout=10 \ cat /etc/redhat-release | \ @@ -82,6 +89,7 @@ while true;do echo "unable to boot a working system within time frame, retrying..." >&2 echo "Cleaning up old image,last status: $(cac update;cac getserver $id | jq -r .status)" eval "$(clear_defer | sed 's/;exit//')" + sleep 15 else echo "got a working system" >&2 break -- cgit v1.3.1 From d73c8df6e4246f34e7a98091bc3c7dab9f90fdde Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 5 Jan 2016 16:07:13 +0100 Subject: k 5 snapraid: is part of upstream --- krebs/5pkgs/snapraid/default.nix | 33 --------------------------- makefu/1systems/omo.nix | 49 +++++++++++++++++++++++----------------- 2 files changed, 28 insertions(+), 54 deletions(-) delete mode 100644 krebs/5pkgs/snapraid/default.nix (limited to 'krebs/5pkgs') diff --git a/krebs/5pkgs/snapraid/default.nix b/krebs/5pkgs/snapraid/default.nix deleted file mode 100644 index 41db0f284..000000000 --- a/krebs/5pkgs/snapraid/default.nix +++ /dev/null @@ -1,33 +0,0 @@ -{stdenv, fetchurl}: -let - s = # Generated upstream information - rec { - baseName="jq"; - version="1.5"; - name="${baseName}-${version}"; - url=https://github.com/stedolan/jq/releases/download/jq-1.5/jq-1.5.tar.gz; - sha256="0g29kyz4ykasdcrb0zmbrp2jqs9kv1wz9swx849i2d1ncknbzln4"; - }; - buildInputs = [ - ]; -in -stdenv.mkDerivation { - inherit (s) name version; - inherit buildInputs; - src = fetchurl { - inherit (s) url sha256; - }; - - # jq is linked to libjq: - configureFlags = [ - "LDFLAGS=-Wl,-rpath,\\\${libdir}" - ]; - meta = { - inherit (s) version; - description = ''A lightweight and flexible command-line JSON processor''; - license = stdenv.lib.licenses.mit ; - maintainers = [stdenv.lib.maintainers.raskin]; - platforms = stdenv.lib.platforms.linux ++ stdenv.lib.platforms.darwin; - }; -} - diff --git a/makefu/1systems/omo.nix b/makefu/1systems/omo.nix index d7d3dba00..65a25a2a1 100644 --- a/makefu/1systems/omo.nix +++ b/makefu/1systems/omo.nix @@ -2,9 +2,18 @@ # your system. Help is available in the configuration.nix(5) man page # and in the NixOS manual (accessible by running ‘nixos-help’). -{ config, pkgs, ... }: - -{ +{ config, pkgs, lib, ... }: +let + byid = dev: "/dev/disk/by-id/" + dev; + keyFile = "/dev/disk/by-id/usb-Verbatim_STORE_N_GO_070B3CEE0B223954-0:0"; + rootDisk = byid "ata-INTEL_SSDSA2M080G2GC_CVPO003402PB080BGN"; + homePartition = byid "ata-INTEL_SSDSA2M080G2GC_CVPO003402PB080BGN-part3"; + cryptDisk0 = byid "ata-ST2000DM001-1CH164_Z240XTT6"; + cryptDisk1 = byid "ata-TP02000GB_TPW151006050068"; + cryptDisk2 = byid "ata-WDC_WD20EARS-00MVWB0_WD-WCAZA5548487"; + # all physical disks + allDisks = [ rootDisk cryptDisk0 cryptDisk1 cryptDisk2 ]; +in { imports = [ # TODO: unlock home partition via ssh @@ -16,35 +25,33 @@ ../2configs/mail-client.nix ]; krebs.build.host = config.krebs.hosts.omo; - services.smartd.devices = [ - { device = "/dev/sda"; } - { device = "/dev/sdb"; } - { device = "/dev/sdc"; } - { device = "/dev/sdd"; } - { device = "/dev/sde"; } - ]; + services.smartd.devices = builtins.map (x: { device = x; }) allDisks; # AMD E350 fileSystems."/home" = { device = "/dev/mapper/home"; fsType = "ext4"; }; - powerManagement.powerUpCommands = '' - for i in a b c d e f g h i;do - ${pkgs.hdparm}/sbin/hdparm -S 100 /dev/sd$i - ${pkgs.hdparm}/sbin/hdparm -B 127 /dev/sd$i - ${pkgs.hdparm}/sbin/hdparm -y /dev/sd$i - ''; + powerManagement.powerUpCommands = lib.concatStrings (map (disk: '' + ${pkgs.hdparm}/sbin/hdparm -S 100 ${disk} + ${pkgs.hdparm}/sbin/hdparm -B 127 ${disk} + ${pkgs.hdparm}/sbin/hdparm -y ${disk} + '') allDisks); boot = { initrd.luks = { - devices = [ - { name = "home"; - device = "/dev/disk/by-uuid/85bff22e-dcbb-4246-b030-faf6c1782995"; + devices = let + usbkey = name: device: { + inherit name device keyFile; keyFileSize = 4096; - keyFile = "/dev/disk/by-id/usb-Verbatim_STORE_N_GO_070B3CEE0B223954-0:0"; } + }; + in [ + (usbkey "home" homePartition) + (usbkey "crypt0" cryptDisk0) + (usbkey "crypt1" cryptDisk1) + (usbkey "crypt2" cryptDisk2) ]; }; - loader.grub.device = "/dev/disk/by-id/ata-INTEL_SSDSA2M080G2GC_CVPO003402PB080BGN"; + loader.grub.device = rootDisk; initrd.availableKernelModules = [ "ahci" -- cgit v1.3.1 From ce10f57b3be0042ccf01ed2605ef9911537552cf Mon Sep 17 00:00:00 2001 From: makefu Date: Sun, 17 Jan 2016 00:30:00 +0100 Subject: k 5 krebszones: add todo/doc --- krebs/5pkgs/krebszones/default.nix | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'krebs/5pkgs') diff --git a/krebs/5pkgs/krebszones/default.nix b/krebs/5pkgs/krebszones/default.nix index f6fd672dc..9230192bd 100644 --- a/krebs/5pkgs/krebszones/default.nix +++ b/krebs/5pkgs/krebszones/default.nix @@ -1,5 +1,10 @@ { lib, pkgs,python3Packages,fetchurl, ... }: +# TODO: Prepare a diff of future and current +## ovh-zone export krebsco.de --config ~/secrets/krebs/cfg.json |sed 's/[ ]\+/ /g' | sort current +## sed 's/[ ]\+/ /g'/etc/zones/krebsco.de | sort > future +## diff future.sorted current.sorted + python3Packages.buildPythonPackage rec { name = "krebszones-${version}"; version = "0.4.4"; -- cgit v1.3.1