fix adv_graphgen, github_listener

5 files changed, 144 insertions, 18 deletions

diff --git a/retiolum/scripts/adv_graphgen/all_the_graphs.sh b/retiolum/scripts/adv_graphgen/all_the_graphs.sh
index 5533c722..8aa0ab21 100755
--- a/retiolum/scripts/adv_graphgen/all_the_graphs.sh
+++ b/retiolum/scripts/adv_graphgen/all_the_graphs.sh
@@ -5,10 +5,12 @@
   cd $(dirname $(readlink -f $0))
   PATH=$PATH:../../../util/bin/
   export LOG_FILE=/var/log/retiolum.log
+  EXTERNAL_FOLDER=/var/www/euer.krebsco.de/graphs/retiolum
+  INTERNAL_FOLDER=/var/www/euer/graphs/retiolum
   begin=`timer`
   export GRAPHITE_HOST="no_omo"
-  (./anonytize.sh /srv/http/pub/graphs/retiolum/ && echo "`date` anonytize done" >> /tmp/build_graph)&
-  (./sanitize.sh /srv/http/priv/graphs/retiolum/ && echo "`date` sanitize done" >> /tmp/build_graph)&
+  (./anonytize.sh $EXTERNAL_FOLDER && echo "`date` anonytize done" >> /tmp/build_graph)&
+  (./sanitize.sh $INTERNAL_FOLDER && echo "`date` sanitize done" >> /tmp/build_graph)&
 #  wait
   graphitec "retiolum.graph.buildtime" "$(timer $begin)"   >> /tmp/build_graph
   echo "`date` end all graphs" >> /tmp/build_graph
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats2json b/retiolum/scripts/adv_graphgen/tinc_stats2json
new file mode 100755
index 00000000..ede19b26
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/tinc_stats2json
@@ -0,0 +1,124 @@
+#!/usr/bin/python
+import subprocess
+import os
+import re
+import sys
+import json
+
+
+
+TINC_NETWORK =os.environ.get("TINC_NETWORK","retiolum")
+
+# is_legacy is the parameter which defines if the tinc config files are handled old fashioned (parse from syslog), 
+# or if the new and hip tincctl should be used
+is_legacy=    os.environ.get("TINC_LEGACY",False)
+SYSLOG_FILE = os.environ.get("LOG_FILE","/var/log/everything.log")
+
+
+# Tags and Delimiters
+TINC_TAG="tinc.%s" % TINC_NETWORK
+BEGIN_NODES = "Nodes:"
+END_NODES = "End of nodes."
+BEGIN_SUBNET = "Subnet list:"
+END_SUBNET = "End of subnet list"
+BEGIN_EDGES = "Edges:"
+END_EDGES = "End of edges."
+
+def get_tinc_block(log_file):
+  """ returns an iterateable block from the given log file (syslog) 
+      This function became obsolete with the introduction of tincctl
+  """
+  from BackwardsReader import BackwardsReader
+  tinc_block = []
+  in_block = False
+  bf = BackwardsReader(log_file)
+  BOL = re.compile(".*tinc.%s\[[0-9]+\]: " % TINC_NETWORK)
+  while True:
+    line = bf.readline()
+    if not line:
+      raise Exception("end of file at log file? This should not happen!")
+    line = BOL.sub('',line).strip()
+
+    if END_SUBNET in line:
+      in_block = True
+
+    if not in_block:
+      continue
+
+    tinc_block.append(line)
+
+    if BEGIN_NODES in line:
+      break
+  return reversed(tinc_block)
+
+def parse_new_input():
+  nodes = {} 
+  pnodes = subprocess.Popen(["tincctl","-n",TINC_NETWORK,"dump","reachable","nodes"], stdout=subprocess.PIPE).communicate()[0]
+  #pnodes = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","reachable","nodes"])
+  for line in pnodes.split('\n'):
+    if not line: continue
+    l = line.split()
+    nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] }
+  psubnets = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","subnets"])
+  for line in psubnets.split('\n'):
+    if not line: continue
+    l = line.split()
+    try:
+      if not nodes[l[2]].get('internal-ip',False):
+         nodes[l[2]]['internal-ip'] = []
+      nodes[l[2]]['internal-ip'].append(l[0].split('#')[0])
+    except KeyError:
+      pass # node does not exist (presumably)
+  pedges = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","edges"])
+  for line in pedges.split('\n'):
+    if not line: continue
+    l = line.split()
+    try:
+      if not nodes[l[0]].has_key('to') :
+        nodes[l[0]]['to'] = []
+      nodes[l[0]]['to'].append(
+          {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] })
+    except KeyError:
+      pass #node does not exist
+  return nodes
+
+def parse_input(log_data):
+  nodes={}
+  for line in log_data:
+    if BEGIN_NODES in line :
+      nodes={}
+      for line in log_data:
+        if END_NODES in line :
+          break
+        l = line.replace('\n','').split() #TODO unhack me
+        nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] }
+    if BEGIN_SUBNET in line :
+      for line in log_data:
+        if END_SUBNET in line :
+          break
+        l = line.replace('\n','').split() 
+        if not nodes[l[2]].get('internal-ip',False):
+           nodes[l[2]]['internal-ip'] = []
+        nodes[l[2]]['internal-ip'].append(l[0].split('#')[0])
+    if BEGIN_EDGES in line :
+      edges = {}
+      for line in log_data:
+        if END_EDGES in line :
+          break
+        l = line.replace('\n','').split() 
+        if not nodes[l[0]].has_key('to') :
+          nodes[l[0]]['to'] = []
+        nodes[l[0]]['to'].append(
+            {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] })
+  return nodes
+
+
+if __name__ == '__main__':
+  import subprocess,time
+  if is_legacy:
+    subprocess.call(["pkill","-SIGUSR2", "tincd"])
+    time.sleep(1)
+    print json.dumps(parse_input((get_tinc_block(SYSLOG_FILE))))
+  else:
+    print json.dumps(parse_new_input())
+
diff --git a/retiolum/scripts/github_listener/github_listener.conf b/retiolum/scripts/github_listener/github_listener.conf
index 88c8ea60..c4f0a8b6 100644
--- a/retiolum/scripts/github_listener/github_listener.conf
+++ b/retiolum/scripts/github_listener/github_listener.conf
@@ -1,4 +1,5 @@
 [program:github_listener]
-command=/opt/github_listener/retiolum/scripts/github_listener/listen 5432
+command=nc -lvv -p 5432 -c "./handle_request /opt/github_listener/retiolum/hosts /var/www/euer.krebsco.de/retiolum/"
 user=tinc
-environment=HOSTFOLDER='/opt/github_listener/retiolum/hosts',WEBDIR='/srv/http/pub/retiolum/'
+directory=/krebs/retiolum/scripts/github_listener/
+autorestart=true
diff --git a/retiolum/scripts/github_listener/handle_request b/retiolum/scripts/github_listener/handle_request
index 2df05ae0..5b42524a 100755
--- a/retiolum/scripts/github_listener/handle_request
+++ b/retiolum/scripts/github_listener/handle_request
@@ -3,14 +3,21 @@
 #   WEBDIR 
 #   HOSTFOLDER
 set -euf
-HOSTFOLDER=${HOSTFOLDER:-../../hosts}
+if [ "x${2:-}" = x ];then
+	echo "usage: $0 HOSTDIRECTORY WEBDIRECTORY"
+	exit 1
+fi
+export HOSTDIR=${1:-../../hosts}
+WEBDIR=${2:-/var/www/euer.krebsco.de/retiolum/}
+echo "sorry for keeping you waiting, please be patient"
+
 cd $(dirname $(readlink -f $0))
-cd "$HOSTFOLDER"
+
+cd "$HOSTDIR"
 git pull origin master  >&2
+echo "First step done"
 cd - >&2
-
-echo "sorry for keeping you waiting, please be patient"
-../../bin/create-supernode-tar
+../../bin/create-supernode-tar $WEBDIR
 echo "almost done..."
-../../bin/create-host-tar
+../../bin/create-host-tar $WEBDIR
 echo "Thank you for your patience!"
diff --git a/retiolum/scripts/github_listener/listen b/retiolum/scripts/github_listener/listen
deleted file mode 100755
index 30f0c4b5..00000000
--- a/retiolum/scripts/github_listener/listen
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh
-PORT=${1:-5432}
-# eloop pattern
-cd $(dirname $(readlink -f $0))
-while sleep 1; do
-  nc -lvv -p $PORT -e ./handle_request && logger "successfully built retiolum hosts tarball"
-
-done