Merge branch 'master' of github.com:krebscode/painload

22 files changed, 111 insertions, 28 deletions

diff --git a/.gitmodules b/.gitmodules
index a8070de9..4a06e547 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,6 +1,3 @@
-[submodule "submodules/github/visionmedia/query"]
-	path = submodules/github/visionmedia/query
-	url = https://github.com/visionmedia/query
 [submodule "submodules/github/tmpvar/jsdom"]
 	path = submodules/github/tmpvar/jsdom
 	url = https://github.com/tmpvar/jsdom
diff --git a/Reaktor/IRC/asybot.py b/Reaktor/IRC/asybot.py
index df758ed6..2cb533ea 100755
--- a/Reaktor/IRC/asybot.py
+++ b/Reaktor/IRC/asybot.py
@@ -2,7 +2,7 @@
 #
 # //Reaktor/IRC/asybot.py
 #
-
+from translate_colors import translate_colors
 def is_executable(x):
   import os
   return os.path.exists(x) and os.access(x, os.X_OK)
@@ -24,6 +24,9 @@ formatter = logging.Formatter( '%(filename)s: %(levelname)s: %(message)s')
 hdlr.setFormatter(formatter)
 log.addHandler(hdlr)
 
+# s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g -- removes color codes
+
+
 class asybot(asychat):
   def __init__(self, server, port, nickname, targets, **kwargs):
     asychat.__init__(self)
@@ -110,6 +113,7 @@ class asybot(asychat):
     def PRIVMSG(text):
       for line in self.wrapper.wrap(text):
         msg = 'PRIVMSG %s :%s' % (','.join(params), line)
+        log.info(msg)
         self.push(msg)
         sleep(1)
 
@@ -151,7 +155,7 @@ class asybot(asychat):
           return
         pid = p.pid
         for line in iter(p.stdout.readline,""):
-          PRIVMSG(line)
+          PRIVMSG(translate_colors(line))
           log.debug('%s stdout: %s' % (pid, line)) 
         p.wait()
         elapsed = time() - start
diff --git a/Reaktor/IRC/index b/Reaktor/IRC/index
index 50022ec9..cc2652fe 100755
--- a/Reaktor/IRC/index
+++ b/Reaktor/IRC/index
@@ -3,4 +3,4 @@ set -xeuf
 
 # cd //Reaktor
 cd $(dirname $(readlink -f $0))/..
-host=irc.freenode.net target='#krebsco' python IRC/asybot.py "$@"
+host=irc.freenode.net target='#krebs' python IRC/asybot.py "$@"
diff --git a/Reaktor/IRC/translate_colors.py b/Reaktor/IRC/translate_colors.py
new file mode 100644
index 00000000..bd716618
--- /dev/null
+++ b/Reaktor/IRC/translate_colors.py
@@ -0,0 +1,31 @@
+
+
+COLOR_MAP = {
+    "\x1b[0m" :  "\x0F", # reset
+    "\x1b[37m" : "\x0300",
+    "\x1b[30m" : "\x0301",
+    "\x1b[34m" : "\x0302",
+    "\x1b[32m" : "\x0303",
+    "\x1b[31m" : "\x0304",
+    "\x1b[33m" : "\x0305",
+    "\x1b[35m" : "\x0306",
+    "\x1b[33m" : "\x0307",
+    "\x1b[33m" : "\x0308",
+    "\x1b[32m" : "\x0309",
+    "\x1b[36m" : "\x0310",
+    "\x1b[36m" : "\x0311",
+    "\x1b[34m" : "\x0312",
+    "\x1b[31m" : "\x0313",
+    "\x1b[30m" : "\x0314",
+    "\x1b[37m" : "\x0315",
+    "\x1b[1m" :  "\x02", # bold on
+    "\x1b[22m" : "\x02" # bold off
+  }
+def translate_colors (line):
+  for color,replace in COLOR_MAP.items():
+    line = line.replace(color,replace)
+  return line
+
+if __name__ == "__main__":
+  import sys
+  print (translate_colors(sys.stdin.read()))
diff --git a/Reaktor/commands/whatweb b/Reaktor/commands/whatweb
index 84130d5c..afe20360 100755
--- a/Reaktor/commands/whatweb
+++ b/Reaktor/commands/whatweb
@@ -2,6 +2,6 @@
 #wrapper for WhatWeb
 here=$(dirname `readlink -f $0`)
 whatweb_bin="$here/../repos/whatweb/whatweb"
-[ -e "$whatweb_bin" ] || ( echo "!! Whatweb app does not exist" && exit 1 )
-[ "balls$1" = "balls" ] && ( echo "!! no host given" && exit 1)
-exec $whatweb_bin -a 3 "$1"
+[ ! -e "$whatweb_bin" ] &&  echo "!! Whatweb app does not exist" && exit 1 
+[ -z "${1:-}" ] &&  echo "!! no host given" && exit 1
+exec $whatweb_bin -a 3 "$1" 2>&1
diff --git a/Reaktor/repos/consolidate_dns/index b/Reaktor/repos/consolidate_dns/index
index 1a0dd81e..3dd42fbd 100755
--- a/Reaktor/repos/consolidate_dns/index
+++ b/Reaktor/repos/consolidate_dns/index
@@ -9,7 +9,7 @@ import tempfile
 os.chdir (os.path.dirname (os.path.realpath (sys.argv[0])))
 dnsrecon_enabled = False
 DNSRECON = "../dnsrecon/dnsrecon.py"
-dnsrecon_wordlist="namelist.txt"
+dnsrecon_wordlist="../dnsrecon/namelist.txt"
 silent=open("/dev/null","w")
 gxfr_enabled = False
 GXFR = "../gxfr/gxfr.py"
@@ -42,7 +42,8 @@ else:
 if dnsrecon_enabled:
   dnsrecon_tmp = tempfile.NamedTemporaryFile(delete=False).name
   print ("Starting dnsrecon, this may take some time")
-  p = Popen([DNSRECON,"-d",DOMAIN,"--csv",dnsrecon_tmp,'-D',dnsrecon_wordlist,"-t","brt,srv,axfr","--skip"],stdout=silent,stderr=silent)
+  #print (" ".join([DNSRECON,"-d",DOMAIN,"--csv",dnsrecon_tmp,'-D',dnsrecon_wordlist,"-t","brt,srv,axfr","--skip"]))
+  p = Popen([DNSRECON,"-d",DOMAIN,"--csv",dnsrecon_tmp,'-D',dnsrecon_wordlist,"-t","brt,srv,axfr"] ,stdout=silent,stderr=silent)
   p.wait()
   reader = csv.reader(open(dnsrecon_tmp))
   for row in reader:
diff --git a/Reaktor/repos/whatweb b/Reaktor/repos/whatweb
-Subproject daab5f21f13024ee8ec47e88f668c5308d6b59d
+Subproject 0918a0d9b75df77f9c3e9eb360b6b22824582a2
diff --git a/retiolum/hosts/machine b/retiolum/hosts/machine
new file mode 100644
index 00000000..4927fc84
--- /dev/null
+++ b/retiolum/hosts/machine
@@ -0,0 +1,11 @@
+Subnet = 10.243.60.31
+Subnet = 42:698d:4e02:4d70:b080:acdd:513d:70aa
+
+-----BEGIN RSA PUBLIC KEY-----
+MIIBCgKCAQEAvkLboZ6wRALd++ntUXfyzCT9G1pWSJNJhrdut8dPfz/+IIbx4thz
+tbq5apWQRaHj6IILMiQqpfUkhbfz3WS2YP62f8nAzKLKB0zzRAJ1lQjoZOXQseQJ
+Ydyf9dEDhRtnSnOwsmSDEch/2KhgCj+fdMcnbcoAg3PYJGzsz2ykEtoh80Rv1IQa
+tW285CP2GooRp1gwy3WKL6at/uW6D4/tTIimHML5JbLKj7mH+3nOyrhRGyZP1b9s
+XtdkePuaQKrIjmv4rEIYx2taFmmQp7XpC2m4Vdoy7WdIzR3WTgWo546IOygY1KIW
+fDOH+3UoG5oI6y4hNNa7+NH8DpmdtzXYnQIDAQAB
+-----END RSA PUBLIC KEY-----
diff --git a/retiolum/hosts/ohz b/retiolum/hosts/ohz
new file mode 100644
index 00000000..e452743b
--- /dev/null
+++ b/retiolum/hosts/ohz
@@ -0,0 +1,10 @@
+Subnet = 10.243.118.171/32
+Subnet = 42:a06d:7412:809a:b74a:8052:daba:c99f/128
+-----BEGIN RSA PUBLIC KEY-----
+MIIBCgKCAQEA7vZFyHtBC9WbXTKcJ2mXxTsZnZqGrDzP7PVtkaBQfTT6J2Qtct5i
+0klA8yvXHUeVdt+hho7rISX4LJr+RDVdhU4ZgrcyJ3rR3moRGzLUV2VLroc1Mnbs
+kkK1mowNk/jZpf6XyRpGL+NFMCZexmfjTdMaMLhzoRbA6w/ffPSSuDZdbG2F5bMk
+BmF6biPcS9Z652ePXh9ViUUKBpLTHQvgK5/iZjI6ik/eit50jrjO6MapUVP/7qob
+VeXE7Zos3UuHLiKegN68VbFQp4qu7jNH4jRun3Pm/Zd/OaGCREIDnfyIyauDNkaT
+QUEL+h0zsM+t2rLT08Wo/sdNX16iMrs9FwIDAQAB
+-----END RSA PUBLIC KEY-----
diff --git a/retiolum/hosts/shabruecke_a0e95c28 b/retiolum/hosts/shabruecke_a0e95c28
new file mode 100644
index 00000000..7b9cbfa0
--- /dev/null
+++ b/retiolum/hosts/shabruecke_a0e95c28
@@ -0,0 +1,12 @@
+Subnet = 10.243.206.121
+Subnet = 42:4687:e87e:41af:e84b:c52f:ccfb:d2f7
+
+-----BEGIN RSA PUBLIC KEY-----
+MIIBCgKCAQEAzN5IRY/9kN8ix5yW2uv1Ltj5rKx1deloB4C/nBsPToA9E/ojASkR
+7V2NDpJGj9QTP4/8ivi0lI+BNOrlHMKxaVvxy/LBmqDQ8ccMQX/4MNGhv1d09iu/
+SBuko/f6lYHm4y71fQlgVuDPS8RDRY/2+008L6rq72RTxuvHZm7smyxDNoUTKWFK
+iJfCxV6nLPiUt3ioEzTMDKAZ7xbOofsTbAxe5Uh4jc9wr3XYkN3y6KHMWDyjWYAo
+WHN0sdEkWONrJvS2b4L3Vl7Mw80Sg1HjmOEJmygY63bDgfP1UnH/ezJxX79RLeZi
+VNF1fD0rJPN937fkYchOT4HJP+RvydB5DwIDAQAB
+-----END RSA PUBLIC KEY-----
+
diff --git a/retiolum/hosts/wookBox b/retiolum/hosts/wookBox
deleted file mode 100644
index 5f1fac86..00000000
--- a/retiolum/hosts/wookBox
+++ /dev/null
@@ -1,10 +0,0 @@
-Subnet = 10.243.129.48
-Subnet = 42:b4f7:9cc9:b129:47a2:fb17:704a:9242
------BEGIN RSA PUBLIC KEY-----
-MIIBCgKCAQEA1Sh6ue0vNr6MU80p04d93k50Fb1nMob3JB/c2OnGQb+QTKiaSFmS
-gyUT1V7UFgcxsXlnvpGpez0MwSW908PyhUS9urO7bedi9O4jKy0fD4sKyRXY4ob7
-R1Z45bMLaRj/25IYqrr/cecDb9/dyMlVU0CiUQY/O9hJvuuUEWYhJO0ubHU2eMoY
-I7cgx7FaxgxVJH1g3u/Ol+Q73oX3HHUt2qk3SuSDi4JklMY+9wysW7GfrcpK3h5o
-jX3J9X7nCMitIstGPNrjIoKXWNH1Dbzwq0yW3XOo3TlOtecwG1G5g9Gz+HcdjoHa
-2jL0AjWtm2klOCXnuq5xe2WJ5wjqzzemWQIDAQAB
------END RSA PUBLIC KEY-----
diff --git a/retiolum/hosts/wookBox1 b/retiolum/hosts/wookBox1
new file mode 100644
index 00000000..bdecc6bd
--- /dev/null
+++ b/retiolum/hosts/wookBox1
@@ -0,0 +1,11 @@
+Subnet = 10.243.203.163
+Subnet = 42:710f:fef4:bfda:fc9c:fcce:266d:62f1
+Address = 188.40.39.66
+-----BEGIN RSA PUBLIC KEY-----
+MIIBCgKCAQEArpm7/ni2lkizLos7j3Y1L9Qrsp0r4G/5T50CCt9fstIrSM+SO23E
+iTTHMBjBWAN9DXuP1VFKttu3SGTmKshbSKdvizs+eg3jlM4+KJRrUwrKTD5O/bfe
+rH4HI8HH9CPyYBEdvxDg4lK+QHzSAZKVFADSYTy7GKb5MR0vFa32AW0yzHLMyPvS
+z8fDg4SaS6gfc3LA7X0cDTVhfB+ulgsLomG4Lwi9XuRnLmn0Ax0nh9HU7GmEtt4l
+Ju/3XMyybbAur73MKThlXMLIWkHiQBL7j1A+04kG59lWNi2YB9VuxbVTEvCZp+pR
+5RRgKz05BL4e/JD9N2pvs+YmqrvL2a3t8QIDAQAB
+-----END RSA PUBLIC KEY-----
diff --git a/retiolum/scripts/adv_graphgen/DEPS b/retiolum/scripts/adv_graphgen/DEPS
new file mode 100644
index 00000000..f14c0aa6
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/DEPS
@@ -0,0 +1,2 @@
+graphviz
+imagemagick
diff --git a/retiolum/scripts/adv_graphgen/anonytize.sh b/retiolum/scripts/adv_graphgen/anonytize.sh
index 2e2045e4..4aad6993 100755
--- a/retiolum/scripts/adv_graphgen/anonytize.sh
+++ b/retiolum/scripts/adv_graphgen/anonytize.sh
@@ -10,7 +10,7 @@ TYPE=svg
 TYPE2=png
 OPENER=/bin/true
 DOTFILE=`mktemp --suffix=anon`
-trap 'rm $DOTFILE' INT TERM KILL
+trap 'rm $DOTFILE' INT TERM KILL EXIT
 sudo -E python tinc_stats2json |\
     python parse_tinc_anon.py> $DOTFILE
 
@@ -28,4 +28,3 @@ convert   $1/retiolum_1.$TYPE  $1/retiolum_1.$TYPE2
 #convert -resize 20% $1/retiolum_2.$TYPE  $1/retiolum_2.$TYPE2
 #convert -resize 20% $1/retiolum_3.$TYPE  $1/retiolum_3.$TYPE2
 #convert -resize 20% $1/retiolum_4.$TYPE  $1/retiolum_4.$TYPE2
-rm $DOTFILE
diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_anon.py b/retiolum/scripts/adv_graphgen/parse_tinc_anon.py
index 21c36e0f..82ee2f2e 100755
--- a/retiolum/scripts/adv_graphgen/parse_tinc_anon.py
+++ b/retiolum/scripts/adv_graphgen/parse_tinc_anon.py
@@ -52,11 +52,13 @@ def write_stat_node(nodes):
   ''' Write a `stats` node in the corner
       This node contains infos about the current number of active nodes and connections inside the network
   '''
+  from time import localtime,strftime
   num_conns = 0
   num_nodes = len(nodes)
   for k,v in nodes.iteritems():
     num_conns+= len(v['to'])
   node_text = "  stats_node [shape=box,label=\"Statistics\\l"
+  node_text += "Build Date  : %s\\l" % strftime("%Y-%m-%d %H:%M:%S",localtime())
   node_text += "Active Nodes: %s\\l" % num_nodes
   node_text += "Connections : %s\\l" % num_conns
   node_text += "\""
diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
index e5bd96a8..e6a67e2c 100755
--- a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
+++ b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
@@ -54,6 +54,7 @@ def write_stat_node(nodes):
   ''' Write a `stats` node in the corner
       This node contains infos about the current number of active nodes and connections inside the network
   '''
+  from time import localtime,strftime
   num_conns = 0
   num_nodes = len(nodes)
   try: 
@@ -64,6 +65,7 @@ def write_stat_node(nodes):
   for k,v in nodes.iteritems():
     num_conns+= len(v['to'])
   node_text = "  stats_node [label=\"Statistics\\l"
+  node_text += "Build Date  : %s\\l" % strftime("%Y-%m-%d %H:%M:%S",localtime())
   node_text += "Active Nodes: %s\\l" % num_nodes
   node_text += "Connections : %s\\l" % num_conns
   node_text += "\""
@@ -172,8 +174,8 @@ def write_node(k,v):
   if v.has_key('num_conns'):
     node += "Num Connects:"+str(v['num_conns'])+"\\l"
   node += "external:"+v['external-ip']+":"+v['external-port']+"\\l"
-  for addr in v.get('internal-ip',['¯\\\\(°_o)/¯']):
-    node += "internal:"+addr+"\\l"
+  for addr in v.get('internal-ip',['dunno lol']): #['¯\\\\(°_o)/¯']):
+    node += "internal:%s\\l"%addr
   node +="\""
 
   # warning if node only has one connection
diff --git a/retiolum/scripts/adv_graphgen/sanitize.sh b/retiolum/scripts/adv_graphgen/sanitize.sh
index 846cc549..c8071dc3 100755
--- a/retiolum/scripts/adv_graphgen/sanitize.sh
+++ b/retiolum/scripts/adv_graphgen/sanitize.sh
@@ -10,7 +10,7 @@ TYPE=svg
 TYPE2=png
 OPENER=/bin/true
 DOTFILE=`mktemp --suffix=san`
-trap 'rm $DOTFILE' INT TERM KILL
+trap 'rm $DOTFILE' INT TERM KILL EXIT
 sudo -E python tinc_stats2json |\
     python parse_tinc_stats.py > $DOTFILE
 
diff --git a/retiolum/scripts/github_listener/github_listener.conf b/retiolum/scripts/github_listener/github_listener.conf
index c4f0a8b6..3ab07a21 100644
--- a/retiolum/scripts/github_listener/github_listener.conf
+++ b/retiolum/scripts/github_listener/github_listener.conf
@@ -3,3 +3,4 @@ command=nc -lvv -p 5432 -c "./handle_request /opt/github_listener/retiolum/hosts
 user=tinc
 directory=/krebs/retiolum/scripts/github_listener/
 autorestart=true
+environment=GRAPHITE_HOST=no_omo
diff --git a/retiolum/scripts/github_listener/handle_request b/retiolum/scripts/github_listener/handle_request
index 5b42524a..211ca776 100755
--- a/retiolum/scripts/github_listener/handle_request
+++ b/retiolum/scripts/github_listener/handle_request
@@ -2,11 +2,15 @@
 # Possible Shell Vars
 #   WEBDIR 
 #   HOSTFOLDER
+
 set -euf
+export GRAPHITE_HOST=${GRAPHITE_HOST:-no_omo}
+PATH=$PATH:../../../util/bin
 if [ "x${2:-}" = x ];then
 	echo "usage: $0 HOSTDIRECTORY WEBDIRECTORY"
 	exit 1
 fi
+NOW=$(timer)
 export HOSTDIR=${1:-../../hosts}
 WEBDIR=${2:-/var/www/euer.krebsco.de/retiolum/}
 echo "sorry for keeping you waiting, please be patient"
@@ -20,4 +24,5 @@ cd - >&2
 ../../bin/create-supernode-tar $WEBDIR
 echo "almost done..."
 ../../bin/create-host-tar $WEBDIR
-echo "Thank you for your patience!"
+echo "Thank you for your patience! I stole $(timer $NOW)ms of your time, sorry about that."
+graphitec 'retiolum.pack.build_time' $(timer $NOW)
diff --git a/retiolum/scripts/tinc_setup/new_install.sh b/retiolum/scripts/tinc_setup/new_install.sh
index 1ff42e54..db154877 100755
--- a/retiolum/scripts/tinc_setup/new_install.sh
+++ b/retiolum/scripts/tinc_setup/new_install.sh
@@ -93,7 +93,7 @@ get_hostname()
     LCOUNTER=0
     if test -e $TSTFILE; then
         while test -e $TSTFILE; do
-            let LCOUNTER=LCOUNTER+1
+            : $((LCOUNTER+=1))
             TSTFILE=$TEMPDIR/hosts/$1$LCOUNTER
         done
         HOSTN=$1$LCOUNTER
diff --git a/submodules/github/visionmedia/query b/submodules/github/visionmedia/query
deleted file mode 160000
-Subproject 02baed4f5eb2d22e3ebb0688243c97fe04e7b53
diff --git a/util/bin/graphitec b/util/bin/graphitec
new file mode 100755
index 00000000..52aa9834
--- /dev/null
+++ b/util/bin/graphitec
@@ -0,0 +1,5 @@
+#!/bin/sh
+H=${GRAPHITE_HOST:-localhost}
+P=${GRAPHITE_PORT:-2003}
+curr=$(date '+%s')
+printf "%s %s %s\n" $1 $2 $curr | nc -q0 $H $P