Merge remote-tracking branch 'lass/20.03' into 20.03

6 files changed, 250 insertions, 101 deletions

diff --git a/krebs/5pkgs/simple/go-shortener/node-packages.nix b/krebs/5pkgs/simple/go-shortener/node-packages.nix
index 526cd5a11..613e31ba0 100644
--- a/krebs/5pkgs/simple/go-shortener/node-packages.nix
+++ b/krebs/5pkgs/simple/go-shortener/node-packages.nix
@@ -1,34 +1,43 @@
-# This file has been generated by node2nix 1.5.3. Do not edit!
+# This file has been generated by node2nix 1.7.0. Do not edit!
 
 {nodeEnv, fetchurl, fetchgit, globalBuildInputs ? []}:
 
 let
   sources = {
-    "double-ended-queue-2.1.0-0" = {
-      name = "double-ended-queue";
-      packageName = "double-ended-queue";
-      version = "2.1.0-0";
+    "denque-1.4.1" = {
+      name = "denque";
+      packageName = "denque";
+      version = "1.4.1";
       src = fetchurl {
-        url = "https://registry.npmjs.org/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz";
-        sha1 = "103d3527fd31528f40188130c841efdd78264e5c";
+        url = "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz";
+        sha512 = "OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ==";
       };
     };
-    "redis-commands-1.3.5" = {
+    "redis-commands-1.5.0" = {
       name = "redis-commands";
       packageName = "redis-commands";
-      version = "1.3.5";
+      version = "1.5.0";
       src = fetchurl {
-        url = "https://registry.npmjs.org/redis-commands/-/redis-commands-1.3.5.tgz";
-        sha512 = "2q8pai3gf0dczb03jjig3mpaz4j2kvw8icpgf8hp4fryr3d6c0mjkvxxdmlyhainybx4zhgqsw9ghc9p986alzdzd2n2p4cxvr8b0by";
+        url = "https://registry.npmjs.org/redis-commands/-/redis-commands-1.5.0.tgz";
+        sha512 = "6KxamqpZ468MeQC3bkWmCB1fp56XL64D4Kf0zJSwDZbVLLm7KFkoIcHrgRvQ+sk8dnhySs7+yBg94yIkAK7aJg==";
       };
     };
-    "redis-parser-2.6.0" = {
+    "redis-errors-1.2.0" = {
+      name = "redis-errors";
+      packageName = "redis-errors";
+      version = "1.2.0";
+      src = fetchurl {
+        url = "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz";
+        sha1 = "eb62d2adb15e4eaf4610c04afe1529384250abad";
+      };
+    };
+    "redis-parser-3.0.0" = {
       name = "redis-parser";
       packageName = "redis-parser";
-      version = "2.6.0";
+      version = "3.0.0";
       src = fetchurl {
-        url = "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz";
-        sha1 = "52ed09dacac108f1a631c07e9b69941e7a19504b";
+        url = "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz";
+        sha1 = "b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4";
       };
     };
   };
@@ -37,40 +46,43 @@ in
   formidable = nodeEnv.buildNodePackage {
     name = "formidable";
     packageName = "formidable";
-    version = "1.2.1";
+    version = "1.2.2";
     src = fetchurl {
-      url = "https://registry.npmjs.org/formidable/-/formidable-1.2.1.tgz";
-      sha512 = "1x0n2rfaiasdjbw1mm7s29z84f68c7iji7lb1gkxvpknvv6q7bwns7z55ijcf4vkh4kvis12rbgaaih49jf9lj53s30mllb1d35bkqn";
+      url = "https://registry.npmjs.org/formidable/-/formidable-1.2.2.tgz";
+      sha512 = "V8gLm+41I/8kguQ4/o1D3RIHRmhYFG4pnNyonvua+40rqcEmT4+V71yaZ3B457xbbgCsCfjSPi65u/W6vK1U5Q==";
     };
     buildInputs = globalBuildInputs;
     meta = {
       description = "A node.js module for parsing form data, especially file uploads.";
-      homepage = https://github.com/felixge/node-formidable;
+      homepage = https://github.com/node-formidable/formidable;
       license = "MIT";
     };
     production = true;
     bypassCache = true;
+    reconstructLock = true;
   };
   redis = nodeEnv.buildNodePackage {
     name = "redis";
     packageName = "redis";
-    version = "2.8.0";
+    version = "3.0.2";
     src = fetchurl {
-      url = "https://registry.npmjs.org/redis/-/redis-2.8.0.tgz";
-      sha512 = "3a3044ax6qdvss83xgjfx10h5q91ls0mwgs3wpsnxcdsiipq3cnmqzsh6glyq0r7vsmpw49jp84c2jnfrhi2bgycrkd9hhhf6ia8lrk";
+      url = "https://registry.npmjs.org/redis/-/redis-3.0.2.tgz";
+      sha512 = "PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==";
     };
     dependencies = [
-      sources."double-ended-queue-2.1.0-0"
-      sources."redis-commands-1.3.5"
-      sources."redis-parser-2.6.0"
+      sources."denque-1.4.1"
+      sources."redis-commands-1.5.0"
+      sources."redis-errors-1.2.0"
+      sources."redis-parser-3.0.0"
     ];
     buildInputs = globalBuildInputs;
     meta = {
-      description = "Redis client library";
-      homepage = https://github.com/NodeRedis/node_redis;
+      description = "A high performance Redis client.";
+      homepage = https://github.com/NodeRedis/node-redis;
       license = "MIT";
     };
     production = true;
     bypassCache = true;
+    reconstructLock = true;
   };
 }
\ No newline at end of file
diff --git a/krebs/5pkgs/simple/go-shortener/update.sh b/krebs/5pkgs/simple/go-shortener/update.sh
index 0c1ecc58c..1a58d0367 100755
--- a/krebs/5pkgs/simple/go-shortener/update.sh
+++ b/krebs/5pkgs/simple/go-shortener/update.sh
@@ -1,2 +1,4 @@
-node2nix -8 -i pkgs.json -c combine.nix
+#!/usr/bin/env nix-shell
+#! nix-shell -i bash -p nodePackages_10_x.node2nix
+node2nix -10 -i pkgs.json -c combine.nix
 rm node-env.nix combine.nix
diff --git a/krebs/5pkgs/simple/grib2json/default.nix b/krebs/5pkgs/simple/grib2json/default.nix
new file mode 100644
index 000000000..2a7959a75
--- /dev/null
+++ b/krebs/5pkgs/simple/grib2json/default.nix
@@ -0,0 +1,19 @@
+{ stdenv, fetchzip, makeWrapper, jre }:
+
+stdenv.mkDerivation {
+  name = "grib2json";
+  src = fetchzip {
+    url = "https://github.com/krebs/grib2json-bin/archive/31efe677b40fe491c988d50f96b59b1b7e2d46f7.zip";
+    sha256 = "1h3yxg270bvac7kaqsbsv4f8nln1i03rfz4cm8cp7llhjj3s6a6b";
+  };
+  installPhase = ''
+    mkdir -p $out/bin
+    cp -r lib $out/
+    cat > $out/bin/grib2json << EOF
+    #!/bin/sh
+    set -x
+    ${jre}/bin/java -jar $out/lib/grib2json-0.8.0-SNAPSHOT.jar "\$@"
+    EOF
+    chmod +x $out/bin/grib2json
+  '';
+}
diff --git a/krebs/5pkgs/simple/irc-announce/default.nix b/krebs/5pkgs/simple/irc-announce/default.nix
index a90814aa9..52cf12862 100644
--- a/krebs/5pkgs/simple/irc-announce/default.nix
+++ b/krebs/5pkgs/simple/irc-announce/default.nix
@@ -25,10 +25,10 @@ pkgs.writeDashBin "irc-announce" ''
   # This is used to see what we send to the irc server. (debug output)
   echo2() { echo "$*"; echo "$*" >&2; }
   cat2() {
-    while read -r line; do
-      echo "$line"
-      echo "$line" >&2
-    done
+    awk '{
+      print $0
+      print $0 > "/dev/stderr"
+    }'
   }
 
   # privmsg_cat transforms stdin to a privmsg
diff --git a/krebs/5pkgs/simple/nomads-cloud/default.nix b/krebs/5pkgs/simple/nomads-cloud/default.nix
new file mode 100644
index 000000000..6e4ace484
--- /dev/null
+++ b/krebs/5pkgs/simple/nomads-cloud/default.nix
@@ -0,0 +1,43 @@
+{ writers, coreutils, grib2json, curl, jq, findutils, imagemagick }:
+writers.writeDashBin "nomads-cloud" ''
+  prefix=$(mktemp -d)
+  grib_path=$prefix.grib
+  json_path=$prefix.json
+  pgm_path=$prefix.pgm
+  png_path="$1"
+
+  mkdir -p "$prefix"
+
+  date=$(${coreutils}/bin/date +%Y%m%d)
+  for hour in 18 12 06 00; do
+    url="https://nomads.ncep.noaa.gov/cgi-bin/filter_gfs_0p25_1hr.pl?file=gfs.t''${hour}z.pgrb2.0p25.anl&lev_entire_atmosphere_%5C%28considered_as_a_single_layer%5C%29=on&var_CWAT=on&leftlon=-180&rightlon=180&toplat=90&bottomlat=-90&dir=%2Fgfs.$date%2F$hour"
+    ${curl}/bin/curl -fsS "$url" > "$grib_path"
+    if [ "$?" -eq 0 ]; then
+      break
+    fi
+  done
+  ${grib2json}/bin/grib2json --data "$grib_path" > "$json_path"
+
+  width=$(${jq}/bin/jq '.[0].header.nx' < "$json_path")
+  height=$(${jq}/bin/jq '.[0].header.ny' < "$json_path")
+
+  # The maximum gray value.  Must be bigger than 0 and less than 65536.
+  maxval=1000
+
+  # pgm - Netpbm grayscale image format
+  # http://netpbm.sourceforge.net/doc/pgm.html
+  {
+    echo P2
+    echo "$width $height"
+    echo "$maxval"
+    cat "$json_path" |
+    ${jq}/bin/jq --argjson maxval "$maxval" -c '
+      ((.[0].data[]) * $maxval | round)
+    ' |
+    ${findutils}/bin/xargs -n "$width"
+  } > "$pgm_path"
+
+  ${imagemagick}/bin/convert -roll +50% "$pgm_path" "$png_path"
+
+  rm -r "$prefix"
+''
diff --git a/krebs/5pkgs/simple/realwallpaper/default.nix b/krebs/5pkgs/simple/realwallpaper/default.nix
index ef83d2e59..057983fec 100644
--- a/krebs/5pkgs/simple/realwallpaper/default.nix
+++ b/krebs/5pkgs/simple/realwallpaper/default.nix
@@ -1,5 +1,5 @@
 { pkgs, ... }:
-pkgs.writeDashBin "generate-wallpaper" ''
+pkgs.writers.writeDashBin "generate-wallpaper" ''
   set -euf
 
   # usage: getimg FILENAME URL
@@ -17,24 +17,28 @@ pkgs.writeDashBin "generate-wallpaper" ''
     fi
   }
 
-  # usage: image_size FILENAME
-  image_size() {
-    identify "$1" | awk '{print$3}'
+  # check if file exists and fetch only if missing
+  fetch_once() {
+    name=$1
+    url=$2
+    test -e "$name" || fetch "$name" "$url"
   }
 
-  # usage: make_mask DST SRC MASK
-  make_layer() {
-    if needs_rebuild "$@"; then
-      echo "make $1 (apply mask)" >&2
-      convert "$2" "$3" -alpha off -compose copy_opacity -composite "$1"
+  fetch_older_min() {
+    min=$1
+    name=$2
+    url=$3
+    if ! test "$(find $name -mmin -$min)"; then
+      fetch "$name" "$url"
     fi
   }
 
-  # usage: flatten DST HILAYER LOLAYER
-  flatten() {
-    if needs_rebuild "$@"; then
-      echo "make $1 (flatten)" >&2
-      composite "$2" "$3" "$1"
+  fetch_older_days() {
+    days=$1
+    name=$2
+    url=$3
+    if ! test "$(find $name -mtime -$days)"; then
+      fetch "$name" "$url"
     fi
   }
 
@@ -48,9 +52,11 @@ pkgs.writeDashBin "generate-wallpaper" ''
     else
       result=1
       for b; do
-        if test "$b" -nt "$a"; then
-          #echo "  $b is newer than $a" >&2
-          result=0
+        if check_type "$b" image; then
+          if test "$b" -nt "$a"; then
+            #echo "  $b is newer than $a" >&2
+            result=0
+          fi
         fi
       done
     fi
@@ -60,34 +66,67 @@ pkgs.writeDashBin "generate-wallpaper" ''
     return $result
   }
 
+  get_neo_url() {
+    url=$1
+    curl -Ss "$url" | grep '3600 x 1800' | sed 's/.*href="\([^"]*\)".*/\1/'
+  }
+
   main() {
     cd "$working_dir"
 
     # fetch source images in parallel
-    fetch nightmap-raw.jpg \
-      "$nightmap_url" &
-    fetch daymap-raw.png \
-      "$daymap_url" &
-    fetch clouds-raw.jpg \
-      "$cloudmap_url" &
-    fetch marker.json \
-      "$marker_url" &
-    wait
+    fetch_once nightmap-raw.jpg \
+      'https://eoimages.gsfc.nasa.gov/images/imagerecords/144000/144898/BlackMarble_2016_3km.jpg' &
+    fetch_once daymap-raw.tif \
+      'https://eoimages.gsfc.nasa.gov/images/imagerecords/57000/57752/land_shallow_topo_8192.tif' &
+
+    fetch_once mercury-raw.svg \
+      'https://upload.wikimedia.org/wikipedia/commons/2/2e/Mercury_symbol.svg' &
+    fetch_once venus-raw.svg \
+      'https://upload.wikimedia.org/wikipedia/commons/6/66/Venus_symbol.svg' &
+    fetch_once mars-raw.svg \
+      'https://upload.wikimedia.org/wikipedia/commons/b/b7/Mars_symbol.svg' &
+    fetch_once jupiter-raw.svg \
+      'https://upload.wikimedia.org/wikipedia/commons/2/26/Jupiter_symbol.svg' &
+    fetch_once saturn-raw.svg \
+      'https://upload.wikimedia.org/wikipedia/commons/7/74/Saturn_symbol.svg' &
+    fetch_once uranus-raw.svg \
+      'https://upload.wikimedia.org/wikipedia/commons/f/f1/Uranus_symbol.svg' &
+    fetch_once neptune-raw.svg \
+      'https://upload.wikimedia.org/wikipedia/commons/4/47/Neptune_symbol.svg' &
+
+    fetch_once krebs-raw.svg \
+      'https://raw.githubusercontent.com/krebs/painload/master/cholerab/bling/krebs_aquarium.svg' &
+
+    fetch_older_min 720 ice-raw.jpg $(get_neo_url \
+      'https://neo.sci.gsfc.nasa.gov/view.php?datasetId=NISE_D') &
+    fetch_older_days 3 snow-raw.jpg $(get_neo_url \
+      'https://neo.sci.gsfc.nasa.gov/view.php?datasetId=MOD10C1_E_SNOW') &
+    fetch_older_days 7 chlora-raw.jpg $(get_neo_url \
+      'https://neo.sci.gsfc.nasa.gov/view.php?datasetId=MY1DMM_CHLORA') &
+    fetch_older_days 3 fire-raw.jpg $(get_neo_url \
+      'https://neo.sci.gsfc.nasa.gov/view.php?datasetId=MOD14A1_E_FIRE') &
 
-    check_type nightmap-raw.jpg image
-    check_type daymap-raw.png image
-    check_type clouds-raw.jpg image
+    # regular fetches
+    fetch marker.json "$marker_url" &
+    fetch sun-raw.jpg 'https://sdo.gsfc.nasa.gov/assets/img/latest/latest_512_0171.jpg' &
 
-    in_size=2048x1024
-    xplanet_out_size=1466x1200
-    out_geometry=1366x768+100+160
+    wait
+
+    # fetch clouds if they are older than 3h
+    if ! test "$(find clouds-raw.png -mmin -180)"; then
+      ${pkgs.nomads-cloud}/bin/nomads-cloud clouds-raw.png
+    fi
 
-    nightsnow_color='#0c1a49'  # nightmap
+    in_size=3600x1800
+    xplanet_out_size=3200x2500
+    out_geometry=3200x1800+0+350
 
     for raw in \
         nightmap-raw.jpg \
-        daymap-raw.png \
-        clouds-raw.jpg \
+        daymap-raw.tif \
+        chlora-raw.jpg \
+        clouds-raw.png \
         ;
     do
       normal=''${raw%-raw.*}.png
@@ -97,49 +136,79 @@ pkgs.writeDashBin "generate-wallpaper" ''
       fi
     done
 
-    # create nightmap-fullsnow
-    if needs_rebuild nightmap-fullsnow.png; then
-      convert -size $in_size xc:$nightsnow_color nightmap-fullsnow.png
+    # remove snow from ice map
+    if needs_rebuild ice.png ice-raw.jpg; then
+      convert ice-raw.jpg -fuzz 20% -fill black -opaque white -colorspace gray -blur 0x6 -scale "$in_size" ice.png
     fi
 
-    # extract daymap-snowmask from daymap-final
-    if needs_rebuild daymap-snowmask.png daymap.png; then
-      convert daymap.png -threshold 95% daymap-snowmask.png
+    if needs_rebuild snow.png snow-raw.jpg; then
+      convert snow-raw.jpg -fuzz 20% -fill '#DEDEDE' -opaque white -scale "$in_size" snow.png
     fi
 
-    # extract nightmap-lightmask from nightmap
-    if needs_rebuild nightmap-lightmask.png nightmap.png; then
-      convert nightmap.png -threshold 25% nightmap-lightmask.png
+    # make fire more red
+    if needs_rebuild fire.png fire-raw.jpg; then
+      convert fire-raw.jpg -fuzz 20% -fill '#ef840c' -opaque white -scale "$in_size" fire.png
     fi
 
-    # create layers
-    make_layer nightmap-snowlayer.png nightmap-fullsnow.png daymap-snowmask.png
-    make_layer nightmap-lightlayer.png nightmap.png nightmap-lightmask.png
+    # cut out sun with alpha transparency
+    if needs_rebuild sun.png sun-raw.jpg; then
+      convert sun-raw.jpg \
+        \( +clone -colorspace HSB -fill white -draw "circle 256,256 256,54" -separate -delete 0,1 \) \
+        -compose copyopacity -composite -crop 512x472+0+20 -scale "100x100" sun.png
+    fi
 
-    # apply layers
-    flatten nightmap-lightsnowlayer.png \
-      nightmap-lightlayer.png \
-      nightmap-snowlayer.png
+    if needs_rebuild krebs.png krebs-raw.svg; then
+      inkscape -z -e krebs.png -w 16 -h 16 krebs-raw.svg
+    fi
 
-    flatten nightmap-final.png \
-      nightmap-lightsnowlayer.png \
-      nightmap.png
+    # -- Planets --
+    for planet in mercury venus mars jupiter saturn uranus neptune; do
+      if needs_rebuild "$planet".png "$planet"-raw.svg; then
+        sed -i 's/#000/#FE8019/g' "$planet"-raw.svg
+        inkscape -z -e "$planet".png -w 40 -h 40 "$planet"-raw.svg
+      fi
+    done
+
+    # -- Daymap --
+
+    # merge with water chlora layer
+    if needs_rebuild daymap-final.png daymap.png fire.png snow.png ice.png chlora.png; then
+      convert daymap.png fire.png -compose lighten -composite daymap-1.png
+      convert daymap-1.png ice.png -compose lighten -composite daymap-2.png
+      convert daymap-2.png snow.png -compose lighten -composite daymap-3.png
+      convert daymap-3.png chlora.png -compose lighten -composite daymap-final.png
+    fi
+
+    # -- Nightmap --
+
+    if needs_rebuild nightmap-final.png nightmap.png fire.png snow.png ice.png chlora.png; then
+      convert nightmap.png fire.png -compose lighten -composite nightmap-1.png
+      convert nightmap-1.png \( -fill black -colorize 70% ice.png \) -compose lighten -composite nightmap-2.png
+      convert nightmap-2.png \( -fill black -colorize 70% snow.png \) -compose lighten -composite nightmap-3.png
+      convert nightmap-3.png \( -fill black -colorize 70% chlora.png \) -compose lighten -composite nightmap-final.png
+    fi
 
     # create marker file from json
     if [ -s marker.json ]; then
-      jq -r 'to_entries[] | @json "\(.value.latitude) \(.value.longitude)"' marker.json > marker_file
+      jq -r 'to_entries[] | @json "\(.value.latitude) \(.value.longitude) image=krebs.png"' marker.json > marker_file
+      echo 'position=sun image=sun.png' >> marker_file
+      echo 'position=moon image=moon.png' >> marker_file
+      echo 'position=mercury image=mercury.png' >> marker_file
+      echo 'position=venus image=venus.png' >> marker_file
+      echo 'position=mars image=mars.png' >> marker_file
+      echo 'position=jupiter image=jupiter.png' >> marker_file
+      echo 'position=saturn image=saturn.png' >> marker_file
+      echo 'position=uranus image=uranus.png' >> marker_file
+      echo 'position=neptune image=neptune.png' >> marker_file
     fi
 
-    # make all unmodified files as final
-    for normal in \
-        daymap.png \
-        clouds.png \
-        ;
-    do
-      final=''${normal%.png}-final.png
-      needs_rebuild $final &&
-        ln $normal $final
-    done
+    # generate moon
+    xplanet -body moon --num_times 1 -origin earth \
+      --transpng moon.png --geometry 50x50 \
+      -config ${pkgs.writeText "moon.config" ''
+        [moon]
+        shade=10
+      ''}
 
     # rebuild every time to update shadow
     xplanet --num_times 1 --geometry $xplanet_out_size \
@@ -149,8 +218,9 @@ pkgs.writeDashBin "generate-wallpaper" ''
         "Earth"
         map=daymap-final.png
         night_map=nightmap-final.png
-        cloud_map=clouds-final.png
-        cloud_threshold=10
+        cloud_map=clouds.png
+        cloud_threshold=1
+        cloud_gamma=10
         shade=15
       ''}
 
@@ -161,8 +231,9 @@ pkgs.writeDashBin "generate-wallpaper" ''
         "Earth"
         map=daymap-final.png
         night_map=nightmap-final.png
-        cloud_map=clouds-final.png
-        cloud_threshold=10
+        cloud_map=clouds.png
+        cloud_threshold=1
+        cloud_gamma=10
         marker_file=marker_file
         shade=15
       ''}
@@ -178,6 +249,8 @@ pkgs.writeDashBin "generate-wallpaper" ''
       convert xplanet-krebs-output.png -crop $out_geometry \
         realwallpaper-krebs-tmp.png
         mv realwallpaper-krebs-tmp.png realwallpaper-krebs.png
+        mkdir -p archive
+        convert realwallpaper-krebs.png archive/"$(date -Is)".jpg
     fi
   }