refactor tinc_stats into a single python module

7 files changed, 381 insertions, 0 deletions

diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py b/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py
new file mode 100755
index 00000000..a1ef13f1
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py
@@ -0,0 +1,58 @@
+#!/usr/bin/python
+# -*- coding: utf8 -*-
+
+import sys,json
+""" TODO: Refactoring needed to pull the edges out of the node structures again,
+it should be easier to handle both structures"""
+DUMP_FILE = "/krebs/db/availability"
+
+def get_all_nodes():
+  import os
+  return os.listdir("/etc/tinc/retiolum/hosts")
+
+def generate_stats():
+  """ Generates availability statistics of the network and nodes
+  """
+  import json
+  jlines = []
+  try:
+    f = open(DUMP_FILE,'r')
+    for line in f:
+      jlines.append(json.loads(line))
+    f.close()
+  except Exception,e:
+    pass
+  all_nodes = {}
+  for k in get_all_nodes():
+    all_nodes[k] = get_node_availability(k,jlines)
+  print ( json.dumps(all_nodes))
+
+def get_node_availability(name,jlines):
+  """ calculates the node availability by reading the generated dump file
+  adding together the uptime of the node and returning the time
+	parms:
+          name - node name
+          jlines - list of already parsed dictionaries node archive
+  """
+  begin = last = current = 0
+  uptime = 0
+  for stat in jlines:
+    if not stat['nodes']:
+      continue
+    ts = stat['timestamp']
+    if not begin:
+      begin = last = ts
+    current = ts
+    if stat['nodes'].get(name,{}).get('to',[]):
+      uptime += current - last
+    else:
+      pass
+    last = ts
+  all_the_time = last - begin
+  try:
+    return uptime/ all_the_time
+  except:
+    return 1
+
+if __name__ == "__main__":
+  generate_stats()
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py b/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py
new file mode 100644
index 00000000..6bdbf43c
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py
@@ -0,0 +1,35 @@
+import sys
+import os
+import string
+
+class BackwardsReader:
+  """ Stripped and stolen from : http://code.activestate.com/recipes/120686-read-a-text-file-backwards/ """
+  def readline(self):
+    while len(self.data) == 1 and ((self.blkcount * self.blksize) < self.size):
+      self.blkcount = self.blkcount + 1
+      line = self.data[0]
+      try:
+        self.f.seek(-self.blksize * self.blkcount, 2) 
+        self.data = string.split(self.f.read(self.blksize) + line, '\n')
+      except IOError:  
+        self.f.seek(0)
+        self.data = string.split(self.f.read(self.size - (self.blksize * (self.blkcount-1))) + line, '\n')
+
+    if len(self.data) == 0:
+      return ""
+
+    line = self.data[-1]
+    self.data = self.data[:-1]
+    return line + '\n'
+
+  def __init__(self, file, blksize=4096):
+    """initialize the internal structures"""
+    self.size = os.stat(file)[6]
+    self.blksize = blksize
+    self.blkcount = 1
+    self.f = open(file, 'rb')
+    if self.size > self.blksize:
+      self.f.seek(-self.blksize * self.blkcount, 2) 
+    self.data = string.split(self.f.read(self.blksize), '\n')
+    if not self.data[-1]:
+      self.data = self.data[:-1]
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py b/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py
new file mode 100755
index 00000000..ba3f6738
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py
@@ -0,0 +1,13 @@
+#!/usr/bin/python3
+# -*- coding: utf8 -*-
+import sys,json
+from Graph import delete_unused_nodes
+
+if __name__ == "__main__":
+  from pygeoip import GeoIP
+  gi = GeoIP("GeoLiteCity.dat")
+  for node,data in delete_unused_nodes(json.load(sys.stdin)).items():
+    try:
+      print ("%s in %s"%(node,gi.record_by_addr(data["external-ip"])["city"]))
+    except:
+      print ("%s myself"%node)
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py b/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py
new file mode 100644
index 00000000..26db3030
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py
@@ -0,0 +1,44 @@
+#!/usr/bin/python
+
+def generate_stats(nodes):
+  """ Generates some statistics of the network and nodes
+  """
+  for k,v in nodes.iteritems():
+    conns = v.get('to',[])
+    for c in conns: #sanitize weights
+      if float(c['weight']) > 9000: c['weight'] = str(9001)
+      elif float(c['weight']) < 0: c['weight'] = str(0)
+    v['num_conns'] = len(conns)
+    v['avg_weight'] = get_node_avg_weight(conns)
+
+def get_node_avg_weight(conns):
+  """ calculates the average weight for the given connections """
+  if not conns:
+    sys.syderr.write("get_node_avg_weight: connection parameter empty")
+    return 9001
+  else:
+    return sum([float(c['weight']) for c in conns])/len(conns)
+
+def delete_unused_nodes(nodes):
+  """ Deletes all the nodes which are currently not connected to the network"""
+  new_nodes = {}
+  for k,v in nodes.iteritems():
+    if v['external-ip'] == "(null)":
+      continue
+    if v.get('to',[]):
+      new_nodes[k] = v
+  for k,v in new_nodes.iteritems():
+    if not [ i for i in v['to'] if i['name'] in new_nodes]:
+      del(k)
+  return new_nodes
+
+def merge_edges(nodes):
+  """ merge back and forth edges into one
+  DESTRUCTS the current structure by deleting "connections" in the nodes
+  """
+  for k,v in nodes.iteritems():
+    for con in v.get('to',[]):
+      for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])):
+        if k == secon['name']:
+          del (nodes[con['name']]['to'][i])
+          con['bidirectional'] = True
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py b/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py
new file mode 100755
index 00000000..644cbc63
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py
@@ -0,0 +1,166 @@
+#!/usr/bin/python
+import subprocess
+import os
+import re
+import sys
+import json
+
+
+TINC_NETWORK =os.environ.get("TINC_NETWORK","retiolum")
+
+# is_legacy is the parameter which defines if the tinc config files are handled old fashioned (parse from syslog), 
+# or if the new and hip tincctl should be used
+
+
+# Tags and Delimiters
+TINC_TAG="tinc.%s" % TINC_NETWORK
+BEGIN_NODES = "Nodes:"
+END_NODES = "End of nodes."
+BEGIN_SUBNET = "Subnet list:"
+END_SUBNET = "End of subnet list"
+BEGIN_EDGES = "Edges:"
+END_EDGES = "End of edges."
+def usage():
+  from sys import argv,exit
+  print("""usage: %s
+This tool dumps all tinc node informations as json
+
+ENVIRONMENT VARIABLES:
+  TINC_NETWORK   The tinc network to dump 
+                      (default: retiolum)
+  LOG_FILE       If legacy tinc is used, defines the log file where tinc stats are dumped in
+                      (default: /var/log/everything.log)
+""" % argv[0])
+  exit(1)
+def debug(func):
+  from functools import wraps
+  @wraps(func)
+  def with_debug(*args,**kwargs):
+    print( func.__name__ + " (args: %s | kwargs %s)"% (args,kwargs))
+    return func(*args,**kwargs)
+  return with_debug
+
+
+def get_tinc_log_file():
+  # TODO parse logfile from somewhere
+  return os.environ.get("LOG_FILE","/var/log/everything.log")
+
+
+def parse_tinc_stats():
+  import subprocess
+  from time import sleep
+  from distutils.spawn import find_executable as which
+  #newest tinc
+  if which("tinc"):
+    return parse_new_input("tinc")
+  #new tinc
+  elif which("tincctl"):
+    return parse_new_input("tincctl")
+  #old tinc
+  elif which("tincd"):
+    # TODO refactor me
+    subprocess.call(["pkill","-SIGUSR2", "tincd"])
+    sleep(1)
+    return parse_input(get_tinc_block(get_tinc_log_file()))
+  #no tinc
+  else:
+    raise Exception("no tinc executable found!")
+  
+#@debug
+def get_tinc_block(log_file):
+  """ returns an iterateable block from the given log file (syslog) 
+      This function became obsolete with the introduction of tincctl
+  """
+  from BackwardsReader import BackwardsReader
+  tinc_block = []
+  in_block = False
+  bf = BackwardsReader(log_file)
+  BOL = re.compile(".*tinc.%s\[[0-9]+\]: " % TINC_NETWORK)
+  while True:
+    line = bf.readline()
+    if not line:
+      raise Exception("end of file at log file? This should not happen!")
+    line = BOL.sub('',line).strip()
+
+    if END_SUBNET in line:
+      in_block = True
+
+    if not in_block:
+      continue
+    tinc_block.append(line)
+
+    if BEGIN_NODES in line:
+      break
+  return reversed(tinc_block)
+
+def parse_new_input(tinc_bin):
+  nodes = {} 
+  pnodes = subprocess.Popen([tinc_bin,"-n",TINC_NETWORK,"dump","reachable","nodes"], stdout=subprocess.PIPE).communicate()[0]
+  #pnodes = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","reachable","nodes"])
+  for line in pnodes.split('\n'):
+    if not line: continue
+    l = line.split()
+    nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] }
+  psubnets = subprocess.check_output([tinc_bin,"-n",TINC_NETWORK,"dump","subnets"])
+  for line in psubnets.split('\n'):
+    if not line: continue
+    l = line.split()
+    try:
+      if not nodes[l[2]].get('internal-ip',False):
+         nodes[l[2]]['internal-ip'] = []
+      nodes[l[2]]['internal-ip'].append(l[0].split('#')[0])
+    except KeyError:
+      pass # node does not exist (presumably)
+  pedges = subprocess.check_output([tinc_bin,"-n",TINC_NETWORK,"dump","edges"])
+  for line in pedges.split('\n'):
+    if not line: continue
+    l = line.split()
+    try:
+      if not nodes[l[0]].has_key('to') :
+        nodes[l[0]]['to'] = []
+      nodes[l[0]]['to'].append(
+          {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] })
+    except KeyError:
+      pass #node does not exist
+  return nodes
+
+#@debug
+def parse_input(log_data):
+  nodes={}
+  for line in log_data:
+    if BEGIN_NODES in line :
+      nodes={}
+      for line in log_data:
+        if END_NODES in line :
+          break
+        l = line.replace('\n','').split() #TODO unhack me
+        nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] }
+    if BEGIN_SUBNET in line :
+      for line in log_data:
+        if END_SUBNET in line :
+          break
+        l = line.replace('\n','').split() 
+        if not nodes[l[2]].get('internal-ip',False):
+           nodes[l[2]]['internal-ip'] = []
+        nodes[l[2]]['internal-ip'].append(l[0].split('#')[0])
+    if BEGIN_EDGES in line :
+      edges = {}
+      for line in log_data:
+        if END_EDGES in line :
+          break
+        l = line.replace('\n','').split() 
+        if not nodes[l[0]].has_key('to') :
+          nodes[l[0]]['to'] = []
+        nodes[l[0]]['to'].append(
+            {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] })
+  return nodes
+
+
+if __name__ == '__main__':
+  # TODO refactor me
+  from sys import argv
+  if len(argv) > 1:
+    usage()
+  else:
+    print json.dumps(parse_tinc_stats())
+
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py b/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py
new file mode 100644
index 00000000..ae0fae8f
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py
@@ -0,0 +1,59 @@
+#!/usr/bin/python
+
+def find_potential_super(path="/etc/tinc/retiolum/hosts"):
+  import os
+  import re
+
+  needle_addr = re.compile("Address\s*=\s*(.*)")
+  needle_port = re.compile("Port\s*=\s*(.*)")
+  for f in os.listdir(path):
+    with open(path+"/"+f) as of:
+      addrs = []
+      port = "655"
+
+      for line in of.readlines():
+
+        addr_found = needle_addr.match(line)
+        if addr_found:
+          addrs.append(addr_found.group(1))
+
+        port_found = needle_port.match(line)
+        if port_found:
+          port = port_found.group(1)
+      
+      if addrs : yield (f ,[(addr ,int(port)) for addr in addrs])
+
+def try_connect(addr):
+  try:
+    from socket import socket,AF_INET,SOCK_STREAM
+    s = socket(AF_INET,SOCK_STREAM)
+    s.settimeout(2)
+    s.connect(addr)
+    s.settimeout(None)
+    s.close()
+    return addr
+  except Exception as e:
+    pass
+    #return ()
+
+def check_one_super(ha):
+    host,addrs = ha
+    valid_addrs = []
+    for addr in addrs:
+      ret = try_connect(addr)
+      if ret: valid_addrs.append(ret)
+    if valid_addrs: return (host,valid_addrs)
+
+def check_all_the_super(path="/etc/tinc/retiolum/hosts"):
+  from multiprocessing import Pool
+  p = Pool(20)
+  return filter(None,p.map(check_one_super,find_potential_super(path)))
+
+
+
+if __name__ == "__main__":
+  """
+  usage
+  """
+  for host,addrs in check_all_the_super():
+    print host,addrs
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py b/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py
new file mode 100644
index 00000000..62f541d1
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py
@@ -0,0 +1,6 @@
+import Availability
+import BackwardsReader
+import Log2JSON
+import Supernodes
+import Geo
+import Graph