Merge remote-tracking branch 'gum/master'

4 files changed, 126 insertions, 450 deletions

diff --git a/makefu/3modules/buildbot/master.nix b/makefu/3modules/buildbot/master.nix
deleted file mode 100644
index 09edac94d..000000000
--- a/makefu/3modules/buildbot/master.nix
+++ /dev/null
@@ -1,263 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-with lib;
-let
-  buildbot = pkgs.buildbot;
-  buildbot-master-config = pkgs.writeText "buildbot-master.cfg" ''
-    # -*- python -*-
-    from buildbot.plugins import *
-    import re
-
-    c = BuildmasterConfig = {}
-
-    c['slaves'] = []
-    # TODO: template potential buildslaves
-    # TODO: set password?
-    slavenames= [ 'testslave' ]
-    for i in slavenames:
-      c['slaves'].append(buildslave.BuildSlave(i, "krebspass"))
-
-    c['protocols'] = {'pb': {'port': 9989}}
-
-    ####### Build Inputs
-    stockholm_repo = 'http://cgit.gum/stockholm'
-    c['change_source'] = []
-    c['change_source'].append(changes.GitPoller(
-            stockholm_repo,
-            workdir='stockholm-poller', branch='master',
-            project='stockholm',
-            pollinterval=120))
-
-    ####### Build Scheduler
-    # TODO: configure scheduler
-    c['schedulers'] = []
-
-    # test the master real quick
-    fast = schedulers.SingleBranchScheduler(
-                                change_filter=util.ChangeFilter(branch="master"),
-                                name="fast-master-test",
-                                builderNames=["fast-tests"])
-
-    force = schedulers.ForceScheduler(
-                                name="force",
-                                builderNames=["full-tests"])
-
-    # files everyone depends on or are part of the share branch
-    def shared_files(change):
-      r =re.compile("^((krebs|share)/.*|Makefile|default.nix)")
-      for file in change.files:
-        if r.match(file):
-          return True
-      return False
-
-    full = schedulers.SingleBranchScheduler(
-                                change_filter=util.ChangeFilter(branch="master"),
-                                fileIsImportant=shared_files,
-                                name="full-master-test",
-                                builderNames=["full-tests"])
-    c['schedulers'] = [ fast, force, full ]
-    ###### The actual build
-    # couple of fast steps:
-    f = util.BuildFactory()
-    ## fetch repo
-    grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
-    f.addStep(grab_repo)
-
-    # the dependencies which are used by the test script
-    deps = [ "gnumake", "jq" ]
-    nixshell = ["nix-shell", "-p" ] + deps + [ "--run" ]
-    def addShell(f,**kwargs):
-      f.addStep(steps.ShellCommand(**kwargs))
-
-    addShell(f,name="centos7-eval",env={"LOGNAME": "shared",
-                  "get" : "krebs.deploy",
-                  "filter" : "json"
-                 },
-             command=nixshell + ["make -s eval system=test-centos7"])
-
-    addShell(f,name="wolf-eval",env={"LOGNAME": "shared",
-                  "get" : "krebs.deploy",
-                  "filter" : "json"
-                 },
-             command=nixshell + ["make -s eval system=wolf"])
-
-    c['builders'] = []
-    c['builders'].append(
-        util.BuilderConfig(name="fast-tests",
-          slavenames=slavenames,
-          factory=f))
-
-    # TODO slow build
-    c['builders'].append(
-        util.BuilderConfig(name="full-tests",
-          slavenames=slavenames,
-          factory=f))
-
-    ####### Status of Builds
-    c['status'] = []
-
-    from buildbot.status import html
-    from buildbot.status.web import authz, auth
-    # TODO: configure if http is wanted
-    authz_cfg=authz.Authz(
-        # TODO: configure user/pw
-        auth=auth.BasicAuth([("krebs","bob")]),
-        gracefulShutdown = False,
-        forceBuild = 'auth',
-        forceAllBuilds = 'auth',
-        pingBuilder = False,
-        stopBuild = False,
-        stopAllBuilds = False,
-        cancelPendingBuild = False,
-    )
-    # TODO: configure nginx
-    c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
-
-    from buildbot.status import words
-    ${optionalString (cfg.irc.enable) ''
-      irc = words.IRC("${cfg.irc.server}", "krebsbuild",
-                      # TODO: multiple channels
-                      channels=["${cfg.irc.channel}"],
-                      notify_events={
-                        #'success': 1,
-                        #'failure': 1,
-                        'exception': 1,
-                        'successToFailure': 1,
-                        'failureToSuccess': 1,
-                      }${optionalString cfg.irc.allowForce ",allowForce=True"})
-      c['status'].append(irc)
-      ''}
-
-    ####### PROJECT IDENTITY
-    c['title'] = "Stockholm"
-    c['titleURL'] = "http://krebsco.de"
-
-    #c['buildbotURL'] = "http://buildbot.krebsco.de/"
-    # TODO: configure url
-    c['buildbotURL'] = "http://vbob:8010/"
-
-    ####### DB URL
-    c['db'] = {
-        'db_url' : "sqlite:///state.sqlite",
-    }
-    ${cfg.extraConfig}
-    '';
-
-  cfg = config.makefu.buildbot.master;
-
-  api = {
-    enable = mkEnableOption "Buildbot Master";
-    workDir = mkOption {
-      default = "/var/lib/buildbot/master";
-      type = types.str;
-      description = ''
-        Path to build bot master directory.
-        Will be created on startup.
-      '';
-    };
-    irc = mkOption {
-      default = {};
-      type = types.submodule ({ config, ... }: {
-        options = {
-          enable = mkEnableOption "Buildbot Master IRC Status";
-          channel = mkOption {
-            default = "nix-buildbot-meetup";
-            type = types.str;
-            description = ''
-              irc channel the bot should connect to
-            '';
-          };
-          allowForce = mkOption {
-            default = false;
-            type = types.bool;
-            description = ''
-              Determines if builds can be forced via IRC
-            '';
-          };
-          nick = mkOption {
-            default = "nix-buildbot";
-            type = types.str;
-            description = ''
-              nickname for IRC
-            '';
-          };
-          server = mkOption {
-            default = "irc.freenode.net";
-            type = types.str;
-            description = ''
-              Buildbot Status IRC Server to connect to
-            '';
-          };
-        };
-      });
-    };
-
-    extraConfig = mkOption {
-      default = "";
-      type = types.lines;
-      description = ''
-        extra config appended to the generated master.cfg
-      '';
-    };
-  };
-
-  imp = {
-
-    users.extraUsers.buildbotMaster = {
-      uid = genid "buildbotMaster";
-      description = "Buildbot Master";
-      home = cfg.workDir;
-      createHome = false;
-    };
-
-    users.extraGroups.buildbotMaster = {
-      gid = 672626386;
-    };
-
-    systemd.services.buildbotMaster = {
-      description = "Buildbot Master";
-      after = [ "network.target" ];
-      wantedBy = [ "multi-user.target" ];
-      path = [ pkgs.git ];
-      serviceConfig = let
-        workdir="${lib.shell.escape cfg.workDir}";
-        # TODO: check if git is the only dep
-      in {
-        PermissionsStartOnly = true;
-        Type = "forking";
-        PIDFile = "${workdir}/twistd.pid";
-        # TODO: maybe also prepare buildbot.tac?
-        ExecStartPre = pkgs.writeScript "buildbot-master-init" ''
-          #!/bin/sh
-          set -efux
-          if [ ! -e ${workdir} ];then
-            mkdir -p ${workdir}
-            ${buildbot}/bin/buildbot create-master -r -l 10 -f ${workdir}
-          fi
-          # always override the master.cfg
-          cp ${buildbot-master-config} ${workdir}/master.cfg
-          # sanity
-          ${buildbot}/bin/buildbot checkconfig ${workdir}
-
-          # TODO: maybe upgrade? not sure about this
-          #       normally we should write buildbot.tac by our own
-          # ${buildbot}/bin/buildbot upgrade-master ${workdir}
-
-          chmod 700 -R ${workdir}
-          chown buildbotMaster:buildbotMaster -R ${workdir}
-        '';
-        ExecStart = "${buildbot}/bin/buildbot start ${workdir}";
-        ExecStop = "${buildbot}/bin/buildbot stop ${workdir}";
-        ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}";
-        PrivateTmp = "true";
-        User = "buildbotMaster";
-        Restart = "always";
-        RestartSec = "10";
-      };
-    };
-  };
-in
-{
-  options.makefu.buildbot.master = api;
-  config = mkIf cfg.enable imp;
-}
diff --git a/makefu/3modules/buildbot/slave.nix b/makefu/3modules/buildbot/slave.nix
deleted file mode 100644
index 7c9ea79c0..000000000
--- a/makefu/3modules/buildbot/slave.nix
+++ /dev/null
@@ -1,185 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-with lib;
-let
-  buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" ''
-    import os
-
-    from buildslave.bot import BuildSlave
-    from twisted.application import service
-
-    basedir = '${cfg.workDir}'
-    rotateLength = 10000000
-    maxRotatedFiles = 10
-
-    application = service.Application('buildslave')
-
-    from twisted.python.logfile import LogFile
-    from twisted.python.log import ILogObserver, FileLogObserver
-    logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength,
-                                  maxRotatedFiles=maxRotatedFiles)
-    application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
-
-    buildmaster_host = '${cfg.masterhost}'
-    # TODO: masterport?
-    port = 9989
-    slavename = '${cfg.username}'
-    passwd = '${cfg.password}'
-    keepalive = 600
-    usepty = 0
-    umask = None
-    maxdelay = 300
-    allow_shutdown = None
-
-    ${cfg.extraConfig}
-
-    s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir,
-                  keepalive, usepty, umask=umask, maxdelay=maxdelay,
-                  allow_shutdown=allow_shutdown)
-    s.setServiceParent(application)
-    '';
-  default-packages = [ pkgs.git pkgs.bash ];
-  cfg = config.makefu.buildbot.slave;
-
-  api = {
-    enable = mkEnableOption "Buildbot Slave";
-
-    workDir = mkOption {
-      default = "/var/lib/buildbot/slave";
-      type = types.str;
-      description = ''
-        Path to build bot slave directory.
-        Will be created on startup.
-      '';
-    };
-
-    masterhost = mkOption {
-      default = "localhost";
-      type = types.str;
-      description = ''
-        Hostname/IP of the buildbot master
-      '';
-    };
-
-    username = mkOption {
-      type = types.str;
-      description = ''
-        slavename used to authenticate with master
-      '';
-    };
-
-    password = mkOption {
-      type = types.str;
-      description = ''
-        slave password used to authenticate with master
-      '';
-    };
-
-    contact = mkOption {
-      default = "nix slave <buildslave@${config.networking.hostName}>";
-      type = types.str;
-      description = ''
-        contact to be announced by buildslave
-      '';
-    };
-
-    description = mkOption {
-      default = "Nix Generated BuildSlave";
-      type = types.str;
-      description = ''
-        description for hostto be announced by buildslave
-      '';
-    };
-
-    packages = mkOption {
-      default = [ pkgs.git ];
-      type = with types; listOf package;
-      description = ''
-        packages which should be in path for buildslave
-      '';
-    };
-
-    extraEnviron = mkOption {
-      default = {};
-      example = {
-        NIX_PATH = "nixpkgs=/path/to/my/nixpkgs";
-      };
-      type = types.attrsOf types.str;
-      description = ''
-        extra environment variables to be provided to the buildslave service
-        if you need nixpkgs, e.g. for running nix-shell you can set NIX_PATH here.
-      '';
-    };
-
-    extraConfig = mkOption {
-      default = "";
-      type = types.lines;
-      example = ''
-        port = 443
-        keepalive = 600
-      '';
-      description = ''
-        extra config evaluated before calling BuildSlave init in .tac file
-      '';
-    };
-  };
-
-  imp = {
-
-    users.extraUsers.buildbotSlave = {
-      uid = genid "buildbotSlave";
-      description = "Buildbot Slave";
-      home = cfg.workDir;
-      createHome = false;
-    };
-
-    users.extraGroups.buildbotSlave = {
-      gid = 1408105834;
-    };
-
-    systemd.services."buildbotSlave-${cfg.username}-${cfg.masterhost}" = {
-      description = "Buildbot Slave for ${cfg.username}@${cfg.masterhost}";
-      after = [ "network.target" ];
-      wantedBy = [ "multi-user.target" ];
-      path = default-packages ++ cfg.packages;
-
-      environment = {
-          NIX_REMOTE="daemon";
-      } // cfg.extraEnviron;
-
-      serviceConfig = let
-        workdir = "${lib.shell.escape cfg.workDir}";
-        contact = "${lib.shell.escape cfg.contact}";
-        description = "${lib.shell.escape cfg.description}";
-        buildbot = pkgs.buildbot-slave;
-        # TODO:make this
-      in {
-        PermissionsStartOnly = true;
-        Type = "forking";
-        PIDFile = "${workdir}/twistd.pid";
-        # TODO: maybe also prepare buildbot.tac?
-        ExecStartPre = pkgs.writeScript "buildbot-master-init" ''
-          #!/bin/sh
-          set -efux
-          mkdir -p ${workdir}/info
-          cp ${buildbot-slave-init} ${workdir}/buildbot.tac
-          echo ${contact} > ${workdir}/info/admin
-          echo ${description} > ${workdir}/info/host
-
-          chown buildbotSlave:buildbotSlave -R ${workdir}
-          chmod 700 -R ${workdir}
-        '';
-        ExecStart = "${buildbot}/bin/buildslave start ${workdir}";
-        ExecStop = "${buildbot}/bin/buildslave stop ${workdir}";
-        PrivateTmp = "true";
-        User = "buildbotSlave";
-        Restart = "always";
-        RestartSec = "10";
-      };
-    };
-  };
-in
-{
-  options.makefu.buildbot.slave = api;
-  config = mkIf cfg.enable imp;
-}
diff --git a/makefu/3modules/default.nix b/makefu/3modules/default.nix
index ffbf54cc0..218c9138e 100644
--- a/makefu/3modules/default.nix
+++ b/makefu/3modules/default.nix
@@ -2,8 +2,7 @@ _:
 
 {
   imports = [
-    ./buildbot/master.nix
-    ./buildbot/slave.nix
+    ./snapraid.nix
   ];
 }
 
diff --git a/makefu/3modules/snapraid.nix b/makefu/3modules/snapraid.nix
new file mode 100644
index 000000000..fbdf50219
--- /dev/null
+++ b/makefu/3modules/snapraid.nix
@@ -0,0 +1,125 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  # returns dirname without / , used as disk name
+  dname = dir: replaceChars ["/"] [""] (head (reverseList (splitString "/" dir)));
+  snapraid-conf = ''
+    # Disks
+    ${concatMapStringsSep "\n" (d: "disk ${dname d} ${d}")  cfg.disks}
+    # Parity
+    ${optionalString (cfg.parity != "") "parity ${cfg.parity}/snapraid.parity"}
+
+    # content on Disks
+    ${optionalString cfg.contentOnDisks
+      concatMapStringsSep "\n" (d: "content ${d}/snapraid.content")  cfg.disks}
+
+    # content on Parity
+    ${optionalString (cfg.contentOnParity && cfg.parity != "")
+      "content ${cfg.parity}/snapraid.content"}
+    # Default content file
+    content ${cfg.defaultContentFile}
+
+    # Extra Configuration
+    ${cfg.extraConfig}
+  '';
+  cfg = config.makefu.snapraid;
+
+  out = {
+    options.makefu.snapraid = api;
+    config = mkIf cfg.enable imp;
+  };
+
+  api = {
+    enable = mkEnableOption "snapraid";
+
+    timerConfig = mkOption {
+      type = types.unspecified;
+      description = ''
+        Start snapraid service
+      '';
+      default = {
+        OnCalendar = "daily";
+      };
+    };
+    disks = mkOption {
+      type = with types;listOf str;
+      description = ''
+        Disks to protect. Each disk is a path to the mounted directory of the
+        disk.
+      '';
+    };
+    parity = mkOption {
+      type = types.str;
+      description = ''
+        Folder to store parity file.
+        Set to empty string if you want to configure the parity yourself in
+        extraConfig.
+
+        All extra parity files (2,3,z, etc...) should be configured via
+        extraConfig.
+      '';
+    };
+    contentOnDisks = mkOption {
+      type = types.bool;
+      default = true;
+      description = ''
+        Store Content file on each Disk to protect.
+        Set this to false if you do not want this behavior to apply.
+      '';
+    };
+    contentOnParity = mkOption {
+      type = types.bool;
+      default = true;
+      description = ''
+        Store Content file on parity Disk.
+        Set this to false if you do not want this behavior to apply.
+      '';
+    };
+    defaultContentFile = mkOption {
+      type = types.str;
+      default = "/var/cache/snapraid.content";
+      description = ''
+        Path to default content file
+        Set to empty string if this content file should be written.
+      '';
+    };
+    extraConfig = mkOption {
+      type = types.string;
+      default = "";
+      description = ''
+        Extra configuration to be appended to the snapraid conf file.
+        You can configure extra Parity files as well as extra content files.
+        See `man snapraid` for additional configuration
+      '';
+    };
+  };
+
+  imp = {
+    environment.systemPackages = [
+      # for scrubbing,fixing
+      pkgs.snapraid
+    ];
+    environment.etc."snapraid.conf".text = snapraid-conf;
+    systemd.timers.snapraid-sync = {
+      description = "snapraid sync timer";
+      wantedBy = [ "timers.target" ];
+      timerConfig = cfg.timerConfig;
+    };
+    systemd.services.snapraid-sync = {
+      description = "Snapraid sync service";
+      after = [ "network.target" "local-fs.target" ];
+
+      serviceConfig = {
+        Type = "simple";
+        ExecStartPre = pkgs.writeScript "Snapraid-sync-init" ''
+          #! /bin/sh
+          ${optionalString (cfg.defaultContentFile != "")
+            "mkdir -p $(dirname ${cfg.defaultContentFile})"}
+        '';
+        ExecStart = "${pkgs.snapraid}/bin/snapraid sync";
+      };
+    };
+  };
+in out