begin packaging of tinc_stats

19 files changed, 192 insertions, 141 deletions

diff --git a/retiolum/scripts/adv_graphgen/.gitignore b/retiolum/scripts/adv_graphgen/.gitignore
new file mode 100644
index 00000000..fbee25a5
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/.gitignore
@@ -0,0 +1,5 @@
+bin/
+include/
+lib/
+*.egg-info/
+GeoLiteCity.dat
diff --git a/retiolum/scripts/adv_graphgen/README b/retiolum/scripts/adv_graphgen/README.md
index 082e0f2b..082e0f2b 100644
--- a/retiolum/scripts/adv_graphgen/README
+++ b/retiolum/scripts/adv_graphgen/README.md
diff --git a/retiolum/scripts/adv_graphgen/all_the_graphs.sh b/retiolum/scripts/adv_graphgen/all_the_graphs.sh
deleted file mode 100755
index 9f3bf82b..00000000
--- a/retiolum/scripts/adv_graphgen/all_the_graphs.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/bash
-
-(
-  echo "`date` begin all graphs" >> /tmp/build_graph
-  cd $(dirname $(readlink -f $0))
-  PATH=$PATH:../../../util/bin/
-  EXTERNAL_FOLDER=/var/www/euer.krebsco.de/graphs/retiolum
-  INTERNAL_FOLDER=/var/www/euer/graphs/retiolum
-  begin=`timer`
-  export GEOCTIYDB="$PWD/GeoLiteCity.dat"
-  (python tinc_stats/Log2JSON.py | python tinc_stats/Geo.py > $INTERNAL_FOLDER/marker.json)&
-  (./anonytize.sh $EXTERNAL_FOLDER && echo "`date` anonytize done" >> /tmp/build_graph)&
-  (./sanitize.sh $INTERNAL_FOLDER && echo "`date` sanitize done" >> /tmp/build_graph)&
-#  wait
-  echo "`date` end all graphs" >> /tmp/build_graph
-)&
diff --git a/retiolum/scripts/adv_graphgen/anonytize.sh b/retiolum/scripts/adv_graphgen/anonytize.sh
deleted file mode 100755
index 04a68869..00000000
--- a/retiolum/scripts/adv_graphgen/anonytize.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/sh
-set -euf
-cd $(dirname `readlink -f $0`)
-GRAPH_SETTER1=dot
-GRAPH_SETTER2=circo
-GRAPH_SETTER3='neato -Goverlap=prism '
-GRAPH_SETTER4=sfdp
-TYPE=svg
-TYPE2=png
-OPENER=/bin/true
-DOTFILE=`mktemp --suffix=anon`
-trap 'rm $DOTFILE' INT TERM KILL EXIT
-sudo -E python tinc_stats/Log2JSON.py |\
-    python tinc_stats/Graph.py anonymous> $DOTFILE
-
-
-i=1
-for setter in dot circo 'neato -Goverlap=prism ' sfdp
-do
-  tmpgraph=`mktemp --tmpdir=$1`
-  $setter -T$TYPE -o $tmpgraph $DOTFILE
-  chmod go+rx $tmpgraph
-  mv $tmpgraph $1/retiolum_$i.$TYPE
-  i=`expr $i + 1`
-done
-convert   $1/retiolum_1.$TYPE  $1/retiolum_1.$TYPE2
-#convert -resize 20% $1/retiolum_2.$TYPE  $1/retiolum_2.$TYPE2
-#convert -resize 20% $1/retiolum_3.$TYPE  $1/retiolum_3.$TYPE2
-#convert -resize 20% $1/retiolum_4.$TYPE  $1/retiolum_4.$TYPE2
diff --git a/retiolum/scripts/adv_graphgen/sanitize.sh b/retiolum/scripts/adv_graphgen/sanitize.sh
deleted file mode 100755
index f7d0e7e7..00000000
--- a/retiolum/scripts/adv_graphgen/sanitize.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/sh
-set -euf
-cd $(dirname `readlink -f $0`)
-GRAPH_SETTER1=dot
-GRAPH_SETTER2=circo
-GRAPH_SETTER3='neato -Goverlap=prism '
-GRAPH_SETTER4=sfdp
-TYPE=svg
-TYPE2=png
-OPENER=/bin/true
-DOTFILE=`mktemp --suffix=san`
-trap 'rm $DOTFILE' INT TERM KILL EXIT
-sudo -E python tinc_stats/Log2JSON.py |\
-    python tinc_stats/Graph.py complete > $DOTFILE
-
-
-i=1
-for setter in dot circo 'neato -Goverlap=prism ' sfdp
-do
-  tmpgraph=`mktemp --tmpdir=$1`
-  $setter -T$TYPE -o $tmpgraph $DOTFILE
-  chmod go+rx $tmpgraph
-  mv $tmpgraph $1/retiolum_$i.$TYPE
-  i=`expr $i + 1`
-done
-
-convert $1/retiolum_1.$TYPE  $1/retiolum_1.$TYPE2
-#convert -resize 20% $1/retiolum_2.$TYPE  $1/retiolum_2.$TYPE2
-#convert -resize 20% $1/retiolum_3.$TYPE  $1/retiolum_3.$TYPE2
-#convert -resize 20% $1/retiolum_4.$TYPE  $1/retiolum_4.$TYPE2
-rm $DOTFILE
diff --git a/retiolum/scripts/adv_graphgen/scripts/all-the-graphs b/retiolum/scripts/adv_graphgen/scripts/all-the-graphs
new file mode 100755
index 00000000..3368d145
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/scripts/all-the-graphs
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+EXTERNAL_FOLDER=${EXTERNAL_FOLDER:-/var/www/euer.krebsco.de/graphs/retiolum}
+INTERNAL_FOLDER=${INTERNAL_FOLDER:-/var/www/euer/graphs/retiolum}
+GEOCTIYDB="${GEOCITYDB:-}"
+
+if test -n "$GEOCITYDB";then
+    # creates a marker file 
+    # TODO: copy map.html from 
+    tinc-stats2json | add-geodata > "$INTERNAL_FOLDER/marker.json"
+fi
+build-graphs anonymous "$EXTERNAL_FOLDER"
+build-graphs complete "$INTERNAL_FOLDER"
diff --git a/retiolum/scripts/adv_graphgen/scripts/build-graphs b/retiolum/scripts/adv_graphgen/scripts/build-graphs
new file mode 100755
index 00000000..3eb675ec
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/scripts/build-graphs
@@ -0,0 +1,27 @@
+#!/bin/sh
+set -euf
+mode=${1?arg1 must be either 'anonymous' or 'complete'}
+out=${2?arg2 must be set to an outdir}
+# for creating all the graphs
+TYPE=svg
+# for creating real pictures
+TYPE2=png
+
+DOTFILE=`mktemp`
+trap 'rm "$DOTFILE"' INT TERM KILL EXIT
+
+tinc-stats2json | tinc-build-graph  "$mode" > "$DOTFILE"
+
+i=1
+for setter in dot circo 'neato -Goverlap=prism ' sfdp
+do
+  tmpgraph=`mktemp --tmpdir=$1`
+  # first build, then move to avoid half-built graphs
+  "$setter" -T$TYPE -o "$tmpgraph" "$DOTFILE" ||:
+  chmod go+rx "$tmpgraph"
+  mv "$tmpgraph" "$1/retiolum_$i.$TYPE"
+  i=`expr $i + 1`
+done
+
+convert -limit memory 32Mib -limit map 64Mib "$out/retiolum_1.$TYPE"  "$out/retiolum_1.$TYPE2"
+rm "$DOTFILE"
diff --git a/retiolum/scripts/adv_graphgen/setup.py b/retiolum/scripts/adv_graphgen/setup.py
new file mode 100644
index 00000000..84597def
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/setup.py
@@ -0,0 +1,41 @@
+import sys
+from setuptools import setup
+
+setup(
+    name='tinc_graphs',
+    version='0.2.3',
+
+    description='Create Graphs from tinc Stats',
+    long_description=open("README.md").read(),
+    license='WTFPL',
+    url='http://krebsco.de/',
+    download_url='https://pypi.python.org/pypi/tinc_graphs/',
+
+    author='krebs',
+    author_email='spam@krebsco.de',
+    # you will also need graphviz and imagemagick
+    install_requires = [ 'pygeoip' ],
+
+    packages=['tinc_graphs'],
+    entry_points={
+        'console_scripts' : [
+            'tinc-stats2json = tinc_graphs.Log2JSON:main',
+            'tinc-build-graph = tinc_graphs.Graph:main',
+            'add-geodata = tinc_graphs.Geo:main',
+            'tinc-availability-stats = tinc_graphs.Availability:generate_stats',
+            ]
+        },
+
+    classifiers=[
+        "Intended Audience :: Developers",
+        "Natural Language :: English",
+        "Operating System :: POSIX :: Linux",
+        "Development Status :: 3 - Alpha",
+        "Programming Language :: Python",
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.4",
+        "Programming Language :: Python :: Implementation :: CPython",
+    ],
+)
+
diff --git a/retiolum/scripts/adv_graphgen/tinc_graphs.nix b/retiolum/scripts/adv_graphgen/tinc_graphs.nix
new file mode 100644
index 00000000..b45fde13
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/tinc_graphs.nix
@@ -0,0 +1,25 @@
+with import <nixpkgs> {};
+# nix-build Reaktor.nix
+# result/bin/reaktor
+## or in your env
+# nix-env -i -f tinc_graphs.nix
+
+buildPythonPackage rec {
+  name = "tinc_graphs-${version}";
+  version = "0.2.6";
+  propagatedBuildInputs = with pkgs;[
+    pythonPackages.docopt
+    graphviz
+    imagemagick
+    pythonPackages.pygeoip
+  ];
+  src = fetchurl {
+    url = "";
+    sha256 = "1dksw1s1n2hxvnga6pygkr174dywncr0wiggkrkn1srbn2amh1c2";
+  };
+  meta = {
+    homepage = http://krebsco.de/;
+    description = "Create Graphs from Tinc Stats";
+    license = stdenv.lib.licenses.wtfpl;
+  };
+}
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Availability.py
index 66defa44..42f28ef9 100755
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py
+++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Availability.py
@@ -1,14 +1,14 @@
 #!/usr/bin/python
 # -*- coding: utf8 -*-
 
-import sys,json
+import sys,json,os
 """ TODO: Refactoring needed to pull the edges out of the node structures again,
 it should be easier to handle both structures"""
-DUMP_FILE = "/krebs/db/availability"
+DUMP_FILE = os.environment.get("AVAILABILITY_FILE","tinc-availability.json")
+hostpath=os.environment.get("TINC_HOSTPATH", "/etc/tinc/retiolum/hosts")
 
 def get_all_nodes():
-  import os
-  return os.listdir("/etc/tinc/retiolum/hosts")
+  return os.listdir(hostpath)
 
 def generate_stats():
   """ Generates availability statistics of the network and nodes
@@ -21,11 +21,13 @@ def generate_stats():
       jlines.append(json.loads(line))
     f.close()
   except Exception as e:
-    pass
+    print("Unable to open and parse Availability DB: {} (override with AVAILABILITY_FILE)".format(DUMP_FILE)
+    sys.exit(1)
+
   all_nodes = {}
   for k in get_all_nodes():
     all_nodes[k] = get_node_availability(k,jlines)
-  print ( json.dumps(all_nodes))
+  print (json.dumps(all_nodes))
 
 def get_node_availability(name,jlines):
   """ calculates the node availability by reading the generated dump file
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py b/retiolum/scripts/adv_graphgen/tinc_graphs/BackwardsReader.py
index 6bdbf43c..6bdbf43c 100644
--- a/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py
+++ b/retiolum/scripts/adv_graphgen/tinc_graphs/BackwardsReader.py
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Geo.py
index 038ca9c0..78add918 100755
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py
+++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Geo.py
@@ -1,9 +1,20 @@
 #!/usr/bin/python3
 # -*- coding: utf8 -*-
 import sys,json,os
-from Graph import delete_unused_nodes,resolve_myself
+from .Graph import delete_unused_nodes,resolve_myself
 GEODB=os.environ.get("GEOCITYDB","GeoLiteCity.dat")
 
+def copy_map():
+    from shutil import copytree
+    from os.path import dirname,join,realpath
+    if len(sys.argv) != 2 or sys.argv[1] == "--help" :
+        print("usage: {} <destination>".format(sys.argv[0])
+        print("  copies the map.html file to the <destination>") 
+        sys.exit(1)
+    dstdir=sys.argv[1]
+    copytree(realpath(join(dirname(__file__),'static/map.html')),dstdir)
+
+
 def add_geo(nodes):
   from pygeoip import GeoIP
   gi = GeoIP(GEODB)
@@ -14,8 +25,8 @@ def add_geo(nodes):
     except Exception as e:
       sys.stderr.write(str(e))
       sys.stderr.write("Cannot determine GeoData for %s\n"%k)
-
   return nodes
+
 def add_coords_to_edges(nodes):
   from pygeoip import GeoIP
   gi = GeoIP(GEODB)
@@ -48,7 +59,19 @@ def add_jitter(nodes):
     except Exception as e: pass
   return nodes
 
-if __name__ == "__main__":
+def main():
   import json
-  nodes = add_jitter(add_coords_to_edges(add_geo(resolve_myself(delete_unused_nodes(json.load(sys.stdin))))))
-  print (json.dumps(nodes))
+  try: 
+      with open(GEODB) as f: f.read()
+  except: 
+      print("cannot open {} (GEODB in env)".format(GEODB))
+      sys.exit(1)
+  try:
+    nodes = add_jitter(add_coords_to_edges(add_geo(resolve_myself(delete_unused_nodes(json.load(sys.stdin))))))
+    print (json.dumps(nodes))
+  except Exception as e:
+      print("cannot parse data received via stdin")
+      print(e)
+
+if __name__ == "__main__":
+    main()
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Graph.py
index 2fb09a58..b001780d 100755
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py
+++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Graph.py
@@ -1,12 +1,12 @@
 #!/usr/bin/python
-from BackwardsReader import BackwardsReader
+from .BackwardsReader import BackwardsReader
 import sys,json,os
-from Supernodes import check_all_the_super
-from Services import add_services
-from Availability import get_node_availability
+from .Supernodes import check_all_the_super
+from .Services import add_services
+from .Availability import get_node_availability
 import sys,json
 from time import time
-DUMP_FILE = "/krebs/db/availability"
+DUMP_FILE = os.environment.get("AVAILABILITY_FILE", "tinc-availability.json")
 
 
 def resolve_myself(nodes):
@@ -19,6 +19,7 @@ def resolve_myself(nodes):
              v["external-ip"] = to["addr"]
   return nodes
 
+
 def dump_graph(nodes):
   from time import time
   graph = {}
@@ -209,7 +210,7 @@ def anonymize_nodes(nodes):
     i = str(int(i)+1)
   return newnodes
 
-if __name__ == "__main__":
+def main():
   supernodes= []
   if len(sys.argv) != 2 or  sys.argv[1] not in ["anonymous","complete"]: 
     print("usage: %s (anonymous|complete)")
@@ -247,4 +248,8 @@ if __name__ == "__main__":
 
   print_stat_node(nodes)
   print ('}')
+
+if __name__ == "__main__":
+  main()
+
 # vim: set sw=2:ts=2
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Log2JSON.py
index a81e2bef..7df042fd 100755
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py
+++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Log2JSON.py
@@ -28,10 +28,9 @@ This tool dumps all tinc node informations as json
 ENVIRONMENT VARIABLES:
   TINC_NETWORK   The tinc network to dump 
                       (default: retiolum)
-  LOG_FILE       If legacy tinc is used, defines the log file where tinc stats are dumped in
-                      (default: /var/log/everything.log)
 """ % argv[0])
   exit(1)
+
 def debug(func):
   from functools import wraps
   @wraps(func)
@@ -52,42 +51,18 @@ def parse_tinc_stats():
   elif which("tincctl"):
     return parse_new_input("tincctl")
   #old tinc
+  elif which("tincd"):
+      print("old tincd not supported")
+      sys.exit(1)
   else:
     raise Exception("no tinc executable found!")
-  
-#@debug
-def get_tinc_block(log_file):
-  """ returns an iterateable block from the given log file (syslog) 
-      This function became obsolete with the introduction of tincctl
-  """
-  from BackwardsReader import BackwardsReader
-  tinc_block = []
-  in_block = False
-  bf = BackwardsReader(log_file)
-  BOL = re.compile(".*tinc.%s\[[0-9]+\]: " % TINC_NETWORK)
-  while True:
-    line = bf.readline()
-    if not line:
-      raise Exception("end of file at log file? This should not happen!")
-    line = BOL.sub('',line).strip()
-
-    if END_SUBNET in line:
-      in_block = True
-
-    if not in_block:
-      continue
-    tinc_block.append(line)
 
-    if BEGIN_NODES in line:
-      break
-  return reversed(tinc_block)
 
 def parse_new_input(tinc_bin):
   nodes = {} 
   pnodes = subprocess.Popen(
           [tinc_bin,"-n",TINC_NETWORK,"dump","reachable","nodes"],
           stdout=subprocess.PIPE).communicate()[0].decode()
-  #pnodes = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","reachable","nodes"])
   for line in pnodes.split('\n'):
     if not line: continue
     l = line.split()
@@ -117,9 +92,12 @@ def parse_new_input(tinc_bin):
       pass #node does not exist
   return nodes
 
-if __name__ == '__main__':
+def main():
   from sys import argv
   if len(argv) > 1:
     usage()
   else:
     print (json.dumps(parse_tinc_stats()))
+
+if __name__ == '__main__':
+    main()
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Services.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Services.py
index 6752e116..227e5aba 100644
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Services.py
+++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Services.py
@@ -1,4 +1,5 @@
-services_dir="/home/makefu/r/services"
+import os,sys
+services_dir=os.environment.get("SERIVCES_DIR","/home/reaktor/nag.services")
 def add_services(nodes):
     for k,v in nodes.items():
         n = nodes[k]
@@ -10,8 +11,15 @@ def add_services(nodes):
         except Exception as e:
             n["services"] = ["Error: No Service File!"]
     return nodes
-if __name__ == "__main__":
+def main():
     import json,sys
-    nodes = add_services(json.load(sys.stdin))
-    print (json.dumps(nodes,indent=4))
+    try:
+        nodes = add_services(json.load(sys.stdin))
+        print (json.dumps(nodes,indent=4))
+    except:
+        print("unable to parse json data from stdin")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()
 # vim: set expandtab:ts=4:sw=4
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Supernodes.py
index 7e1f4dae..c8514c66 100755
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py
+++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Supernodes.py
@@ -23,6 +23,7 @@ def find_potential_super(path="/etc/tinc/retiolum/hosts"):
       
       if addrs : yield (f ,[(addr ,int(port)) for addr in addrs])
 
+
 def try_connect(addr):
   try:
     from socket import socket,AF_INET,SOCK_STREAM
@@ -34,7 +35,7 @@ def try_connect(addr):
     return addr
   except Exception as e:
     pass
-    #return ()
+
 
 def check_one_super(ha):
     host,addrs = ha
@@ -44,16 +45,21 @@ def check_one_super(ha):
       if ret: valid_addrs.append(ret)
     if valid_addrs: return (host,valid_addrs)
 
-def check_all_the_super(path="/etc/tinc/retiolum/hosts"):
+
+def check_all_the_super(path):
   from multiprocessing import Pool
   p = Pool(20)
   return filter(None,p.map(check_one_super,find_potential_super(path)))
 
 
+def main():
+  import os
+  hostpath=os.environment.get("TINC_HOSTPATH", "/etc/tinc/retiolum/hosts")
 
-if __name__ == "__main__":
-  """
-  usage
-  """
-  for host,addrs in check_all_the_super():
+  for host,addrs in check_all_the_super(hostpath):
     print("%s %s" %(host,str(addrs)))
+    
+if __name__ == "__main__":
+    main()
+
+# vim: set expandtab:ts=:sw=2
diff --git a/retiolum/scripts/adv_graphgen/tinc_graphs/__init__.py b/retiolum/scripts/adv_graphgen/tinc_graphs/__init__.py
new file mode 100644
index 00000000..414ffe99
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/tinc_graphs/__init__.py
@@ -0,0 +1 @@
+__version__="0.2.3"
diff --git a/retiolum/scripts/adv_graphgen/map.html b/retiolum/scripts/adv_graphgen/tinc_graphs/static/map.html
index ef8a0565..ef8a0565 100644
--- a/retiolum/scripts/adv_graphgen/map.html
+++ b/retiolum/scripts/adv_graphgen/tinc_graphs/static/map.html
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py b/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py
deleted file mode 100644
index f0fc8520..00000000
--- a/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import Availability
-import BackwardsReader
-import Log2JSON
-import Supernodes
-import Geo
-import Graph
-import Services