From 461fe008e72995a42e8546d5dcc46382ca820000 Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 1 Feb 2016 21:58:19 +0100 Subject: ma 1 filepimp: use by-id fs path, snapraid --- makefu/1systems/filepimp.nix | 51 +++++++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/makefu/1systems/filepimp.nix b/makefu/1systems/filepimp.nix index 2d008cee6..fb9324ee9 100644 --- a/makefu/1systems/filepimp.nix +++ b/makefu/1systems/filepimp.nix @@ -1,10 +1,14 @@ -# Edit this configuration file to define what should be installed on -# your system. Help is available in the configuration.nix(5) man page -# and in the NixOS manual (accessible by running ‘nixos-help’). - -{ config, pkgs, ... }: - -{ +{ config, pkgs, lib, ... }: +let + byid = dev: "/dev/disk/by-id/" + dev; + part1 = disk: disk + "-part1"; + rootDisk = byid "ata-SanDisk_SDSSDP064G_140237402890"; + jDisk0 = byid "ata-ST4000DM000-1F2168_Z303HVSG"; + jDisk1 = byid "ata-ST4000DM000-1F2168_Z3040NEA"; + jDisk2 = byid "ata-WDC_WD40EFRX-68WT0N0_WD-WCC4E0621363"; + jDisk3 = byid "ata-TOSHIBA_MD04ACA400_156GK89OFSBA"; + allDisks = [ rootDisk jDisk0 jDisk1 jDisk2 jDisk3 ]; +in { imports = [ # Include the results of the hardware scan. ../2configs/fs/single-partition-ext4.nix @@ -12,16 +16,9 @@ ../2configs/smart-monitor.nix ]; krebs.build.host = config.krebs.hosts.filepimp; - services.smartd.devices = [ - { device = "/dev/sda"; } - { device = "/dev/sdb"; } - { device = "/dev/sdc"; } - { device = "/dev/sdd"; } - { device = "/dev/sde"; } - ]; # AMD N54L boot = { - loader.grub.device = "/dev/sde"; + loader.grub.device = rootDisk; initrd.availableKernelModules = [ "ahci" @@ -40,4 +37,28 @@ zramSwap.enable = true; zramSwap.numDevices = 2; + + makefu.snapraid = let + toMedia = name: "/media/" + name; + in { + enable = true; + # todo combine creation when enabling the mount point + disks = map toMedia [ "j0" "j1" "j2" ]; + parity = toMedia "par0"; + }; + # TODO: refactor, copy-paste from omo + services.smartd.devices = builtins.map (x: { device = x; }) allDisks; + powerManagement.powerUpCommands = lib.concatStrings (map (disk: '' + ${pkgs.hdparm}/sbin/hdparm -S 100 ${disk} + ${pkgs.hdparm}/sbin/hdparm -B 127 ${disk} + ${pkgs.hdparm}/sbin/hdparm -y ${disk} + '') allDisks); + fileSystems = let + xfsmount = name: dev: + { "/media/${name}" = { device = dev; fsType = "xfs"; }; }; + in + (xfsmount "j0" (part1 jDisk0)) + // (xfsmount "j1" (part1 jDisk1)) + // (xfsmount "j2" (part1 jDisk2)) + // (xfsmount "par0" (part1 jDisk3)); } -- cgit v1.2.3 From 40b13f240888be643e19939ceef79483aeb07ca5 Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 1 Feb 2016 21:58:54 +0100 Subject: ma 1 gum: host update.connector.one --- makefu/1systems/gum.nix | 1 + makefu/2configs/nginx/update.connector.one.nix | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 makefu/2configs/nginx/update.connector.one.nix diff --git a/makefu/1systems/gum.nix b/makefu/1systems/gum.nix index ac7524506..c4dfbf4b7 100644 --- a/makefu/1systems/gum.nix +++ b/makefu/1systems/gum.nix @@ -15,6 +15,7 @@ in { ../2configs/git/cgit-retiolum.nix ../2configs/mattermost-docker.nix ../2configs/nginx/euer.test.nix + ../2configs/nginx/update.connector.one.nix ../2configs/exim-retiolum.nix ../2configs/urlwatch.nix diff --git a/makefu/2configs/nginx/update.connector.one.nix b/makefu/2configs/nginx/update.connector.one.nix new file mode 100644 index 000000000..eb39a1668 --- /dev/null +++ b/makefu/2configs/nginx/update.connector.one.nix @@ -0,0 +1,26 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + hostname = config.krebs.build.host.name; + external-ip = head config.krebs.build.host.nets.internet.addrs4; +in { + krebs.nginx = { + enable = mkDefault true; + servers = { + omo-share = { + listen = [ "${external-ip}:80" ]; + server-names = [ + "update.connector.one" + "firmware.connector.one" + ]; + locations = singleton (nameValuePair "/" '' + autoindex on; + root /var/www/update.connector.one; + sendfile on; + gzip on; + ''); + }; + }; + }; +} -- cgit v1.2.3 From 44e0c5153ca6a65ee130f30ea8466906deedcada Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 1 Feb 2016 22:01:41 +0100 Subject: ma 1 omo: add shares --- makefu/1systems/omo.nix | 30 +--------------- makefu/2configs/nginx/omo-share.nix | 34 ------------------- makefu/2configs/omo-share.nix | 68 +++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 63 deletions(-) delete mode 100644 makefu/2configs/nginx/omo-share.nix create mode 100644 makefu/2configs/omo-share.nix diff --git a/makefu/1systems/omo.nix b/makefu/1systems/omo.nix index 19183fea8..e9c51f485 100644 --- a/makefu/1systems/omo.nix +++ b/makefu/1systems/omo.nix @@ -28,7 +28,7 @@ in { ../2configs/smart-monitor.nix ../2configs/mail-client.nix ../2configs/share-user-sftp.nix - ../2configs/nginx/omo-share.nix + ../2configs/omo-share.nix ../3modules ]; networking.firewall.trustedInterfaces = [ "enp3s0" ]; @@ -42,34 +42,6 @@ in { # services.openssh.allowSFTP = false; krebs.build.source.git.nixpkgs.rev = "d0e3cca04edd5d1b3d61f188b4a5f61f35cdf1ce"; - # samba share /media/crypt1/share - users.users.smbguest = { - name = "smbguest"; - uid = config.ids.uids.smbguest; - description = "smb guest user"; - home = "/var/empty"; - }; - services.samba = { - enable = true; - shares = { - winshare = { - path = "/media/crypt1/share"; - "read only" = "no"; - browseable = "yes"; - "guest ok" = "yes"; - }; - }; - extraConfig = '' - guest account = smbguest - map to guest = bad user - # disable printing - load printers = no - printing = bsd - printcap name = /dev/null - disable spoolss = yes - ''; - }; - # copy config from to /var/lib/sabnzbd/ services.sabnzbd.enable = true; systemd.services.sabnzbd.environment.SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; diff --git a/makefu/2configs/nginx/omo-share.nix b/makefu/2configs/nginx/omo-share.nix deleted file mode 100644 index ce85e0442..000000000 --- a/makefu/2configs/nginx/omo-share.nix +++ /dev/null @@ -1,34 +0,0 @@ -{ config, lib, pkgs, ... }: - -with lib; -let - hostname = config.krebs.build.host.name; - # TODO local-ip from the nets config - local-ip = "192.168.1.11"; - # local-ip = head config.krebs.build.host.nets.retiolum.addrs4; -in { - krebs.nginx = { - enable = mkDefault true; - servers = { - omo-share = { - listen = [ "${local-ip}:80" ]; - locations = singleton (nameValuePair "/" '' - autoindex on; - root /media; - limit_rate_after 100m; - limit_rate 5m; - mp4_buffer_size 4M; - mp4_max_buffer_size 10M; - allow all; - access_log off; - keepalive_timeout 65; - keepalive_requests 200; - reset_timedout_connection on; - sendfile on; - tcp_nopush on; - gzip off; - ''); - }; - }; - }; -} diff --git a/makefu/2configs/omo-share.nix b/makefu/2configs/omo-share.nix new file mode 100644 index 000000000..1e0975e1d --- /dev/null +++ b/makefu/2configs/omo-share.nix @@ -0,0 +1,68 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + hostname = config.krebs.build.host.name; + # TODO local-ip from the nets config + local-ip = "192.168.1.11"; + # local-ip = head config.krebs.build.host.nets.retiolum.addrs4; +in { + krebs.nginx = { + enable = mkDefault true; + servers = { + omo-share = { + listen = [ "${local-ip}:80" ]; + locations = singleton (nameValuePair "/" '' + autoindex on; + root /media; + limit_rate_after 100m; + limit_rate 5m; + mp4_buffer_size 4M; + mp4_max_buffer_size 10M; + allow all; + access_log off; + keepalive_timeout 65; + keepalive_requests 200; + reset_timedout_connection on; + sendfile on; + tcp_nopush on; + gzip off; + ''); + }; + }; + }; + + # samba share /media/crypt1/share + users.users.smbguest = { + name = "smbguest"; + uid = config.ids.uids.smbguest; + description = "smb guest user"; + home = "/var/empty"; + }; + services.samba = { + enable = true; + shares = { + winshare = { + path = "/media/crypt1/share"; + "read only" = "no"; + browseable = "yes"; + "guest ok" = "yes"; + }; + usenet = { + path = "/media/crypt0/usenet/dst"; + "read only" = "yes"; + browseable = "yes"; + "guest ok" = "yes"; + }; + }; + extraConfig = '' + guest account = smbguest + map to guest = bad user + # disable printing + load printers = no + printing = bsd + printcap name = /dev/null + disable spoolss = yes + ''; + }; +} -- cgit v1.2.3 From fdc4fa5c98aaabfb31be7e7f219ca2b134172cf9 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 3 Feb 2016 21:17:46 +0100 Subject: cacpanel 0.2.3 -> cac-panel 0.4.4 --- krebs/5pkgs/cac-panel/default.nix | 18 ++++++++++++++++++ krebs/5pkgs/cacpanel/default.nix | 18 ------------------ 2 files changed, 18 insertions(+), 18 deletions(-) create mode 100644 krebs/5pkgs/cac-panel/default.nix delete mode 100644 krebs/5pkgs/cacpanel/default.nix diff --git a/krebs/5pkgs/cac-panel/default.nix b/krebs/5pkgs/cac-panel/default.nix new file mode 100644 index 000000000..fd4799535 --- /dev/null +++ b/krebs/5pkgs/cac-panel/default.nix @@ -0,0 +1,18 @@ +{pkgs, python3Packages, ...}: + +python3Packages.buildPythonPackage rec { + name = "cac-panel-${version}"; + version = "0.4.4"; + + src = pkgs.fetchurl { + url = "https://pypi.python.org/packages/source/c/cac-panel/cac-panel-${version}.tar.gz"; + sha256 = "16bx67fsbgwxciik42jhdnfzxx1xp5b0rimzrif3r7h4fawlnld8"; + }; + + propagatedBuildInputs = with python3Packages; [ + docopt + requests2 + beautifulsoup4 + ]; +} + diff --git a/krebs/5pkgs/cacpanel/default.nix b/krebs/5pkgs/cacpanel/default.nix deleted file mode 100644 index 3df4dffed..000000000 --- a/krebs/5pkgs/cacpanel/default.nix +++ /dev/null @@ -1,18 +0,0 @@ -{pkgs, python3Packages, ...}: - -python3Packages.buildPythonPackage rec { - name = "cacpanel-${version}"; - version = "0.2.3"; - - src = pkgs.fetchurl { - url = "https://pypi.python.org/packages/source/c/cacpanel/cacpanel-${version}.tar.gz"; - sha256 = "1fib7416qqv8yzrj75kxra7ccpz9abqh58b6gkaavws2fa6m3mm8"; - }; - - propagatedBuildInputs = with python3Packages; [ - docopt - requests2 - beautifulsoup4 - ]; -} - -- cgit v1.2.3 From f7d979b21fc0a705105adbbc708645f94af6629c Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 4 Feb 2016 02:48:28 +0100 Subject: s 1 wolf: provide cgit mirror --- krebs/3modules/shared/default.nix | 1 + shared/1systems/wolf.nix | 1 + shared/2configs/cgit-mirror.nix | 41 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 shared/2configs/cgit-mirror.nix diff --git a/krebs/3modules/shared/default.nix b/krebs/3modules/shared/default.nix index 518e46587..91d92857b 100644 --- a/krebs/3modules/shared/default.nix +++ b/krebs/3modules/shared/default.nix @@ -50,6 +50,7 @@ in { addrs6 = ["42:0:0:0:0:0:77:1"]; aliases = [ "wolf.retiolum" + "cgit.wolf.retiolum" ]; tinc.pubkey = '' -----BEGIN RSA PUBLIC KEY----- diff --git a/shared/1systems/wolf.nix b/shared/1systems/wolf.nix index 8cf5be71c..e45195487 100644 --- a/shared/1systems/wolf.nix +++ b/shared/1systems/wolf.nix @@ -12,6 +12,7 @@ in ../2configs/shack-nix-cacher.nix ../2configs/shack-drivedroid.nix ../2configs/buildbot-standalone.nix + ../2configs/cgit-mirror.nix # ../2configs/graphite.nix ]; # use your own binary cache, fallback use cache.nixos.org (which is used by diff --git a/shared/2configs/cgit-mirror.nix b/shared/2configs/cgit-mirror.nix new file mode 100644 index 000000000..5bcfc5818 --- /dev/null +++ b/shared/2configs/cgit-mirror.nix @@ -0,0 +1,41 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + rules = with git;[{ + # user = git-sync; + user = git-sync; + repo = [ stockholm-mirror ]; + perm = push ''refs/*'' [ non-fast-forward create delete merge ]; + }]; + + stockholm-mirror = { + public = true; + name = "stockholm-mirror"; + desc = "mirror for all stockholm branches"; + hooks = { + post-receive = pkgs.git-hooks.irc-announce { + nick = config.networking.hostName; + verbose = false; + channel = "#retiolum"; + server = "cd.retiolum"; + }; + }; + }; + + git-sync = { + name = "git-sync"; + mail = "spam@krebsco.de"; + # TODO put git-sync pubkey somewhere more appropriate + pubkey = ''ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCzUuzyoAhMgJmsiaTVWNSXqcrZNTpKpv0nfFBOMcNXUWEbvfAq5eNpg5cX+P8eoYl6UQgfftbYi06flKK3yJdntxoZKLwJGgJt9NZr8yZTsiIfMG8XosvGNQtGPkBtpLusgmPpu7t2RQ9QrqumBvoUDGYEauKTslLwupp1QeyWKUGEhihn4CuqQKiPrz+9vbNd75XOfVZMggk3j4F7HScatmA+p1EQXWyq5Jj78jQN5ZIRnHjMQcIZ4DOz1U96atwSKMviI1xEZIODYfgoGjjiWYeEtKaLVPtSqtLRGI7l+RNouMfwHLdTWOJSlIdFncfPXC6R19hTll3UHeHLtqLP git-sync''; + }; + +in { + krebs.git = { + enable = true; + root-title = "Shared Repos"; + root-desc = "keep on krebsing"; + inherit rules; + repos.stockholm-mirror = stockholm-mirror; + }; +} -- cgit v1.2.3 From cc1a230fd2742b6ccadd0837d9cf569f246375aa Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 4 Feb 2016 02:55:28 +0100 Subject: k 5 test: cac -> cac-api, cacpanel -> cac-panel --- krebs/5pkgs/test/infest-cac-centos7/default.nix | 6 +++--- krebs/5pkgs/test/infest-cac-centos7/notes | 26 ++++++++++++------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/krebs/5pkgs/test/infest-cac-centos7/default.nix b/krebs/5pkgs/test/infest-cac-centos7/default.nix index 886e250e2..7adb09ca9 100644 --- a/krebs/5pkgs/test/infest-cac-centos7/default.nix +++ b/krebs/5pkgs/test/infest-cac-centos7/default.nix @@ -1,4 +1,4 @@ -{ stdenv, coreutils,makeWrapper, cac, cacpanel, gnumake, gnused, jq, openssh, ... }: +{ stdenv, coreutils,makeWrapper, cac-api, cac-panel, gnumake, gnused, jq, openssh, ... }: stdenv.mkDerivation rec { name = "${shortname}-${version}"; @@ -14,8 +14,8 @@ stdenv.mkDerivation rec { path = stdenv.lib.makeSearchPath "bin" [ coreutils - cac - cacpanel + cac-api + cac-panel gnumake gnused jq diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes index 6bfb6906e..793ef3560 100755 --- a/krebs/5pkgs/test/infest-cac-centos7/notes +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -1,6 +1,6 @@ #! /bin/sh -# nix-shell -p gnumake jq openssh cac cacpanel +# nix-shell -p gnumake jq openssh cac-api cac-panel set -eufx # 2 secrets are required: @@ -40,22 +40,22 @@ defer "rm -r $krebs_secrets" cat > $sec_file <&1) + out=$(cac-api build cpu=1 ram=512 storage=10 os=26 2>&1) if name=$(echo "$out" | jq -r .servername);then id=servername:$name echo "got a working machine, id=$id" @@ -87,7 +87,7 @@ while true;do # die on timeout if ! wait_login_cac $id;then echo "unable to boot a working system within time frame, retrying..." >&2 - echo "Cleaning up old image,last status: $(cac update;cac getserver $id | jq -r .status)" + echo "Cleaning up old image,last status: $(cac-api update;cac-api getserver $id | jq -r .status)" eval "$(clear_defer | sed 's/;exit//')" sleep 15 else @@ -96,17 +96,17 @@ while true;do fi done clear_defer >/dev/null -defer "cac delete $id;$old_trapstr" +defer "cac-api delete $id;$old_trapstr" mkdir -p shared/2configs/temp -cac generatenetworking $id > \ +cac-api generatenetworking $id > \ shared/2configs/temp/networking.nix # new temporary ssh key we will use to log in after infest ssh-keygen -f $krebs_ssh -N "" cp $retiolum_key $krebs_secrets/retiolum.rsa_key.priv # we override the directories for secrets and stockholm # additionally we set the ssh key we generated -ip=$(cac getserver $id | jq -r .ip) +ip=$(cac-api getserver $id | jq -r .ip) cat > shared/2configs/temp/dirs.nix < $krebs_secrets/infest sh -x $krebs_secrets/infest # TODO: generate secrets directory $krebs_secrets for nix import -cac powerop $id reset +cac-api powerop $id reset wait_login(){ # timeout -- cgit v1.2.3 From c9cfaa010dabbd37a329a9690debf7cf7ef5e3ba Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 4 Feb 2016 12:02:11 +0100 Subject: ma: finish merge of new sources v2, nixos compatibility --- makefu/2configs/default.nix | 12 ++++-------- makefu/2configs/wwan.nix | 1 - 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/makefu/2configs/default.nix b/makefu/2configs/default.nix index a98393e2b..2b4e31119 100644 --- a/makefu/2configs/default.nix +++ b/makefu/2configs/default.nix @@ -20,10 +20,10 @@ with lib; build = { target = mkDefault "root@${config.krebs.build.host.name}"; user = config.krebs.users.makefu; - source = { + source = mapAttrs (_: mkDefault) { upstream-nixpkgs = { - url = mkDefault https://github.com/nixos/nixpkgs; - rev = mkDefault "93d8671e2c6d1d25f126ed30e5e6f16764330119"; # unstable @ 2015-01-03, tested on filepimp + url = https://github.com/nixos/nixpkgs; + rev = "93d8671e2c6d1d25f126ed30e5e6f16764330119"; # unstable @ 2015-01-03, tested on filepimp }; secrets = "/home/makefu/secrets/${config.krebs.build.host.name}/"; stockholm = "/home/makefu/stockholm"; @@ -80,11 +80,7 @@ with lib; ]; environment.variables = { - NIX_PATH = with config.krebs.build.source; with dir; with git; - mkForce (concatStringsSep ":" [ - "nixpkgs=${nixpkgs.target-path}" - "${nixpkgs.target-path}" - ]); + NIX_PATH = mkForce "/var/src"; EDITOR = mkForce "vim"; }; diff --git a/makefu/2configs/wwan.nix b/makefu/2configs/wwan.nix index 1e76cd28a..0eb0c97d7 100644 --- a/makefu/2configs/wwan.nix +++ b/makefu/2configs/wwan.nix @@ -1,7 +1,6 @@ _: { - imports = [ ../3modules ]; makefu.umts = { enable = true; modem-device = "/dev/serial/by-id/usb-Lenovo_H5321_gw_2D5A51BA0D3C3A90-if01"; -- cgit v1.2.3 From e402c8ce1d2786abafc1efdc64adca84d174a756 Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 4 Feb 2016 13:07:47 +0100 Subject: k 3 git.nix: flesh out rules description --- krebs/3modules/git.nix | 2 +- makefu/1systems/omo.nix | 1 - makefu/1systems/vbob.nix | 19 ------------------- 3 files changed, 1 insertion(+), 21 deletions(-) diff --git a/krebs/3modules/git.nix b/krebs/3modules/git.nix index 7b28ffca8..11cf21b5f 100644 --- a/krebs/3modules/git.nix +++ b/krebs/3modules/git.nix @@ -92,7 +92,7 @@ let } ''; description = '' - Rules. + access and permission rules for git repositories. ''; }; }; diff --git a/makefu/1systems/omo.nix b/makefu/1systems/omo.nix index d15cc2779..34d5a394d 100644 --- a/makefu/1systems/omo.nix +++ b/makefu/1systems/omo.nix @@ -29,7 +29,6 @@ in { ../2configs/mail-client.nix ../2configs/share-user-sftp.nix ../2configs/omo-share.nix - ../3modules ]; networking.firewall.trustedInterfaces = [ "enp3s0" ]; # udp:137 udp:138 tcp:445 tcp:139 - samba, allowed in local net diff --git a/makefu/1systems/vbob.nix b/makefu/1systems/vbob.nix index 6c8f5ca26..f4a22d720 100644 --- a/makefu/1systems/vbob.nix +++ b/makefu/1systems/vbob.nix @@ -18,25 +18,6 @@ tinc = pkgs.tinc_pre; }; - makefu.buildbot.master = { - enable = false; - irc = { - enable = true; - server = "cd.retiolum"; - channel = "retiolum"; - allowForce = true; - }; - }; - # services.logstash.enable = true; - makefu.buildbot.slave = { - enable = false; - masterhost = "localhost"; - username = "testslave"; - password = "krebspass"; - packages = with pkgs;[ git nix ]; - extraEnviron = { NIX_PATH="nixpkgs=${toString }"; }; - }; - krebs.build.source.nixpkgs = { # url = https://github.com/nixos/nixpkgs; # HTTP Everywhere + libredir -- cgit v1.2.3 From 5be8920fb0262ff703f23ef484c59f4b55a9b015 Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 4 Feb 2016 14:36:48 +0100 Subject: s 2 base: new paths, cosmetics --- shared/2configs/base.nix | 18 +++++++----------- shared/2configs/cgit-mirror.nix | 7 +++---- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/shared/2configs/base.nix b/shared/2configs/base.nix index 5e6072661..dd698ba97 100644 --- a/shared/2configs/base.nix +++ b/shared/2configs/base.nix @@ -16,20 +16,16 @@ with lib; # TODO rename shared user to "krebs" krebs.build.user = mkDefault config.krebs.users.shared; krebs.build.source = { - git.nixpkgs = { + upstream-nixpkgs = mkDefault { url = https://github.com/NixOS/nixpkgs; rev = "d0e3cca"; - target-path = "/var/src/nixpkgs"; - }; - dir.secrets = { - host = config.krebs.current.host; - path = mkDefault "${getEnv "HOME"}/secrets/krebs/${config.krebs.build.host.name}"; - }; - dir.stockholm = { - host = config.krebs.current.host; - path = mkDefault "${getEnv "HOME"}/stockholm"; - target-path = "/var/src/stockholm"; }; + secrets = mkDefault "${getEnv "HOME"}/secrets/krebs/${config.krebs.build.host.name}"; + stockholm = mkDefault "${getEnv "HOME"}/stockholm"; + + nixos-config = "symlink:stockholm/${config.krebs.build.user.name}/1systems/${config.krebs.build.host.name}.nix"; + nixpkgs = symlink:stockholm/nixpkgs; + stockholm-user = "symlink:stockholm/${config.krebs.build.user.name}"; }; networking.hostName = config.krebs.build.host.name; diff --git a/shared/2configs/cgit-mirror.nix b/shared/2configs/cgit-mirror.nix index 5bcfc5818..4ff1902f9 100644 --- a/shared/2configs/cgit-mirror.nix +++ b/shared/2configs/cgit-mirror.nix @@ -2,12 +2,11 @@ with lib; let - rules = with git;[{ - # user = git-sync; - user = git-sync; + rules = with git; singleton { + user = [ git-sync ]; repo = [ stockholm-mirror ]; perm = push ''refs/*'' [ non-fast-forward create delete merge ]; - }]; + }; stockholm-mirror = { public = true; -- cgit v1.2.3 From 4c23e33dea4d9901b64bf287983c43862f4990f2 Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 4 Feb 2016 16:38:21 +0100 Subject: ma 1: refactor buildbot config, add documentation --- shared/1systems/wolf.nix | 2 +- shared/2configs/buildbot-standalone.nix | 150 -------------------------------- shared/2configs/shared-buildbot.nix | 148 +++++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 151 deletions(-) delete mode 100644 shared/2configs/buildbot-standalone.nix create mode 100644 shared/2configs/shared-buildbot.nix diff --git a/shared/1systems/wolf.nix b/shared/1systems/wolf.nix index e45195487..bcfbd6810 100644 --- a/shared/1systems/wolf.nix +++ b/shared/1systems/wolf.nix @@ -11,7 +11,7 @@ in ../2configs/collectd-base.nix ../2configs/shack-nix-cacher.nix ../2configs/shack-drivedroid.nix - ../2configs/buildbot-standalone.nix + ../2configs/shared-buildbot.nix ../2configs/cgit-mirror.nix # ../2configs/graphite.nix ]; diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix deleted file mode 100644 index 9982dd915..000000000 --- a/shared/2configs/buildbot-standalone.nix +++ /dev/null @@ -1,150 +0,0 @@ -{ lib, config, pkgs, ... }: - -{ - networking.firewall.allowedTCPPorts = [ 8010 9989 ]; - krebs.buildbot.master = { - secrets = [ "retiolum-ci.rsa_key.priv" "cac.json" ]; - slaves = { - testslave = "krebspass"; - }; - change_source.stockholm = '' - stockholm_repo = 'http://cgit.gum/stockholm' - cs.append(changes.GitPoller( - stockholm_repo, - workdir='stockholm-poller', branch='master', - project='stockholm', - pollinterval=120)) - ''; - scheduler = { - force-scheduler = '' - sched.append(schedulers.ForceScheduler( - name="force", - builderNames=["full-tests"])) - ''; - fast-tests-scheduler = '' - # test the master real quick - sched.append(schedulers.SingleBranchScheduler( - change_filter=util.ChangeFilter(branch="master"), - treeStableTimer=10, #only test the latest push - name="fast-master-test", - builderNames=["fast-tests"])) - ''; - test-cac-infest-master = '' - # files everyone depends on or are part of the share branch - def shared_files(change): - r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)") - for file in change.files: - if r.match(file): - return True - return False - - sched.append(schedulers.SingleBranchScheduler( - change_filter=util.ChangeFilter(branch="master"), - fileIsImportant=shared_files, - treeStableTimer=60*60, # master was stable for the last hour - name="full-master-test", - builderNames=["full-tests"])) - ''; - }; - builder_pre = '' - # prepare grab_repo step for stockholm - stockholm_repo = "http://cgit.gum.retiolum/stockholm" - grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') - - env = {"LOGNAME": "shared", "NIX_REMOTE": "daemon"} - - # prepare nix-shell - # the dependencies which are used by the test script - deps = [ "gnumake", "jq","nix","rsync", - "(import {}).pkgs.test.infest-cac-centos7" ] - # TODO: --pure , prepare ENV in nix-shell command: - # SSL_CERT_FILE,LOGNAME,NIX_REMOTE - nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ] - - # prepare addShell function - def addShell(factory,**kwargs): - factory.addStep(steps.ShellCommand(**kwargs)) - ''; - builder = { - fast-tests = '' - f = util.BuildFactory() - f.addStep(grab_repo) - addShell(f,name="deploy-eval-centos7",env=env, - command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"]) - - addShell(f,name="deploy-eval-wolf",env=env, - command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"]) - - addShell(f,name="deploy-eval-cross-check",env=env, - command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"]) - - addShell(f,name="instantiate-test-all-modules",env=env, - command=nixshell + \ - ["touch retiolum.rsa_key.priv; \ - nix-instantiate --eval -A \ - users.shared.test-all-krebs-modules.system \ - -I stockholm=. \ - --show-trace \ - -I secrets=. '' \ - --argstr current-user-name shared \ - --argstr current-host-name lol \ - --strict --json"]) - - addShell(f,name="instantiate-test-minimal-deploy",env=env, - command=nixshell + \ - ["nix-instantiate --eval -A \ - users.shared.test-minimal-deploy.system \ - -I stockholm=. \ - -I secrets=. '' \ - --show-trace \ - --argstr current-user-name shared \ - --argstr current-host-name lol \ - --strict --json"]) - - bu.append(util.BuilderConfig(name="fast-tests", - slavenames=slavenames, - factory=f)) - ''; - slow-tests = '' - s = util.BuildFactory() - s.addStep(grab_repo) - - # slave needs 2 files: - # * cac.json - # * retiolum - s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/cac.json", slavedest="cac.json")) - s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/retiolum-ci.rsa_key.priv", slavedest="retiolum.rsa_key.priv")) - - addShell(s, name="infest-cac-centos7",env=env, - sigtermTime=60, # SIGTERM 1 minute before SIGKILL - timeout=10800, # 3h - command=nixshell + ["infest-cac-centos7"]) - - bu.append(util.BuilderConfig(name="full-tests", - slavenames=slavenames, - factory=s)) - ''; - }; - enable = true; - web = { - enable = true; - }; - irc = { - enable = true; - nick = "shared-buildbot"; - server = "cd.retiolum"; - channels = [ "retiolum" ]; - allowForce = true; - }; - }; - - krebs.buildbot.slave = { - enable = true; - masterhost = "localhost"; - username = "testslave"; - password = "krebspass"; - packages = with pkgs;[ git nix ]; - # all nix commands will need a working nixpkgs installation - extraEnviron = { NIX_PATH="nixpkgs=${toString }"; }; - }; -} diff --git a/shared/2configs/shared-buildbot.nix b/shared/2configs/shared-buildbot.nix new file mode 100644 index 000000000..50b279036 --- /dev/null +++ b/shared/2configs/shared-buildbot.nix @@ -0,0 +1,148 @@ +{ lib, config, pkgs, ... }: +# The buildbot config is seilf-contained and provides a way to test "shared" +# configuration (infrastructure to be used by every krebsminister). + +# You can add your own test, test steps as required. Deploy the config on a +# shared host like wolf and everything should be fine. +{ + networking.firewall.allowedTCPPorts = [ 8010 9989 ]; + krebs.buildbot.master = { + secrets = [ "retiolum-ci.rsa_key.priv" "cac.json" ]; + slaves = { + testslave = "krebspass"; + }; + change_source.stockholm = '' + stockholm_repo = 'http://cgit.gum/stockholm' + cs.append(changes.GitPoller( + stockholm_repo, + workdir='stockholm-poller', branch='master', + project='stockholm', + pollinterval=120)) + ''; + scheduler = { + force-scheduler = '' + sched.append(schedulers.ForceScheduler( + name="force", + builderNames=["full-tests"])) + ''; + fast-tests-scheduler = '' + # test the master real quick + sched.append(schedulers.SingleBranchScheduler( + change_filter=util.ChangeFilter(branch="master"), + treeStableTimer=10, #only test the latest push + name="fast-master-test", + builderNames=["fast-tests"])) + ''; + test-cac-infest-master = '' + # files everyone depends on or are part of the share branch + def shared_files(change): + r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)") + for file in change.files: + if r.match(file): + return True + return False + + sched.append(schedulers.SingleBranchScheduler( + change_filter=util.ChangeFilter(branch="master"), + fileIsImportant=shared_files, + treeStableTimer=60*60, # master was stable for the last hour + name="full-master-test", + builderNames=["full-tests"])) + ''; + }; + builder_pre = '' + # prepare grab_repo step for stockholm + stockholm_repo = "http://cgit.gum.retiolum/stockholm" + grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') + + env = {"LOGNAME": "shared", "NIX_REMOTE": "daemon"} + + # prepare nix-shell + # the dependencies which are used by the test script + deps = [ "gnumake", "jq","nix","rsync", + "(import {}).pkgs.test.infest-cac-centos7" ] + # TODO: --pure , prepare ENV in nix-shell command: + # SSL_CERT_FILE,LOGNAME,NIX_REMOTE + nixshell = ["nix-shell", + "-I", "stockholm=.", + "-I", "nixpkgs=/var/src/upstream-nixpkgs", + "-p" ] + deps + [ "--run" ] + + # prepare addShell function + def addShell(factory,**kwargs): + factory.addStep(steps.ShellCommand(**kwargs)) + ''; + builder = { + fast-tests = '' + f = util.BuildFactory() + f.addStep(grab_repo) + for i in [ "test-centos7", "wolf", "test-failing" ]: + addShell(f,name="populate-{}".format(i),env=env, + command=nixshell + ["set -o pipefail;{}( nix-instantiate --arg configuration shared/1systems/{}.nix --eval --readonly-mode --show-trace -A config.krebs.build.populate --strict | jq -r .)".format("!" if "failing" in i else "",i)]) + + addShell(f,name="instantiate-test-all-modules",env=env, + command=nixshell + \ + ["touch retiolum.rsa_key.priv; \ + nix-instantiate --eval -A \ + users.shared.test-all-krebs-modules.system \ + -I stockholm=. \ + --show-trace \ + -I secrets=. '' \ + --strict --json"]) + + addShell(f,name="instantiate-test-minimal-deploy",env=env, + command=nixshell + \ + ["nix-instantiate --eval -A \ + users.shared.test-minimal-deploy.system \ + -I stockholm=. \ + -I secrets=. '' \ + --show-trace \ + --strict --json"]) + + bu.append(util.BuilderConfig(name="fast-tests", + slavenames=slavenames, + factory=f)) + ''; + slow-tests = '' + s = util.BuildFactory() + s.addStep(grab_repo) + + # slave needs 2 files: + # * cac.json + # * retiolum + s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/cac.json", slavedest="cac.json")) + s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/retiolum-ci.rsa_key.priv", slavedest="retiolum.rsa_key.priv")) + + addShell(s, name="infest-cac-centos7",env=env, + sigtermTime=60, # SIGTERM 1 minute before SIGKILL + timeout=10800, # 3h + command=nixshell + ["infest-cac-centos7"]) + + bu.append(util.BuilderConfig(name="full-tests", + slavenames=slavenames, + factory=s)) + ''; + }; + enable = true; + web = { + enable = true; + }; + irc = { + enable = true; + nick = "shared-buildbot"; + server = "cd.retiolum"; + channels = [ "retiolum" ]; + allowForce = true; + }; + }; + + krebs.buildbot.slave = { + enable = true; + masterhost = "localhost"; + username = "testslave"; + password = "krebspass"; + packages = with pkgs;[ git nix ]; + # all nix commands will need a working nixpkgs installation + extraEnviron = { NIX_PATH="/var/src"; }; + }; +} -- cgit v1.2.3