refactor tinc_stats into a single python module

12 files changed, 75 insertions, 206 deletions

diff --git a/retiolum/scripts/adv_graphgen/anonytize.sh b/retiolum/scripts/adv_graphgen/anonytize.sh
index 4aad6993..4ed5c8db 100755
--- a/retiolum/scripts/adv_graphgen/anonytize.sh
+++ b/retiolum/scripts/adv_graphgen/anonytize.sh
@@ -11,7 +11,7 @@ TYPE2=png
 OPENER=/bin/true
 DOTFILE=`mktemp --suffix=anon`
 trap 'rm $DOTFILE' INT TERM KILL EXIT
-sudo -E python tinc_stats2json |\
+sudo -E python tinc_stats/Log2JSON.py |\
     python parse_tinc_anon.py> $DOTFILE
 
 
diff --git a/retiolum/scripts/adv_graphgen/find_super.py b/retiolum/scripts/adv_graphgen/find_super.py
deleted file mode 100644
index ae0fae8f..00000000
--- a/retiolum/scripts/adv_graphgen/find_super.py
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/python
-
-def find_potential_super(path="/etc/tinc/retiolum/hosts"):
-  import os
-  import re
-
-  needle_addr = re.compile("Address\s*=\s*(.*)")
-  needle_port = re.compile("Port\s*=\s*(.*)")
-  for f in os.listdir(path):
-    with open(path+"/"+f) as of:
-      addrs = []
-      port = "655"
-
-      for line in of.readlines():
-
-        addr_found = needle_addr.match(line)
-        if addr_found:
-          addrs.append(addr_found.group(1))
-
-        port_found = needle_port.match(line)
-        if port_found:
-          port = port_found.group(1)
-      
-      if addrs : yield (f ,[(addr ,int(port)) for addr in addrs])
-
-def try_connect(addr):
-  try:
-    from socket import socket,AF_INET,SOCK_STREAM
-    s = socket(AF_INET,SOCK_STREAM)
-    s.settimeout(2)
-    s.connect(addr)
-    s.settimeout(None)
-    s.close()
-    return addr
-  except Exception as e:
-    pass
-    #return ()
-
-def check_one_super(ha):
-    host,addrs = ha
-    valid_addrs = []
-    for addr in addrs:
-      ret = try_connect(addr)
-      if ret: valid_addrs.append(ret)
-    if valid_addrs: return (host,valid_addrs)
-
-def check_all_the_super(path="/etc/tinc/retiolum/hosts"):
-  from multiprocessing import Pool
-  p = Pool(20)
-  return filter(None,p.map(check_one_super,find_potential_super(path)))
-
-
-
-if __name__ == "__main__":
-  """
-  usage
-  """
-  for host,addrs in check_all_the_super():
-    print host,addrs
diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_anon.py b/retiolum/scripts/adv_graphgen/parse_tinc_anon.py
index 82ee2f2e..9041fb35 100755
--- a/retiolum/scripts/adv_graphgen/parse_tinc_anon.py
+++ b/retiolum/scripts/adv_graphgen/parse_tinc_anon.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python
 # -*- coding: utf8 -*-
-from BackwardsReader import BackwardsReader
+from tinc_stats.BackwardsReader import BackwardsReader
+from tinc_stats.Graph import generate_stats,delete_unused_nodes,merge_edges
 import sys,json
 #supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ]
 try:
@@ -19,6 +20,7 @@ except Exception as e:
 """ TODO: Refactoring needed to pull the edges out of the node structures again,
 it should be easier to handle both structures"""
 DUMP_FILE = "/krebs/db/availability"
+
 def write_digraph(nodes):
   """
   writes the complete digraph in dot format
@@ -37,6 +39,7 @@ def write_digraph(nodes):
   print ('}')
 
 def anon_nodes(nodes):
+  #anonymizes all nodes
   i = "0"
   newnodes = {}
   for k,v in nodes.iteritems():
@@ -66,46 +69,6 @@ def write_stat_node(nodes):
   node_text += "]"
   print(node_text)
 
-def generate_stats(nodes):
-  """ Generates some statistics of the network and nodes
-  """
-  for k,v in nodes.iteritems():
-    conns = v.get('to',[])
-    for c in conns: #sanitize weights
-      if float(c['weight']) > 9000: c['weight'] = str(9001)
-      elif float(c['weight']) < 0: c['weight'] = str(0)
-    v['num_conns'] = len(conns)
-    v['avg_weight'] = get_node_avg_weight(conns)
-def get_node_avg_weight(conns):
-  """ calculates the average weight for the given connections """
-  if not conns:
-    sys.syderr.write("get_node_avg_weight: connection parameter empty")
-    return 9001
-  else:
-    return sum([float(c['weight']) for c in conns])/len(conns)
-
-def delete_unused_nodes(nodes):
-  new_nodes = {}
-  for k,v in nodes.iteritems():
-    if v['external-ip'] == "(null)":
-      continue
-    if v.get('to',[]):
-      new_nodes[k] = v
-  for k,v in new_nodes.iteritems():
-    if not [ i for i in v['to'] if i['name'] in new_nodes]:
-      #del(new_nodes[k])
-      del(k)
-  return new_nodes
-def merge_edges(nodes):
-  """ merge back and forth edges into one
-  DESTRUCTS the current structure by deleting "connections" in the nodes
-  """
-  for k,v in nodes.iteritems():
-    for con in v.get('to',[]):
-      for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])):
-        if k == secon['name']:
-          del (nodes[con['name']]['to'][i])
-          con['bidirectional'] = True
 
 
 def write_node(k,v):
@@ -115,13 +78,6 @@ def write_node(k,v):
   """
   
   node = "  "+k #+"[label=\""
-  #node += k+"\\l"
-  #node += "avg weight: %.2f\\l" % v['avg_weight'] 
-  #if v.has_key('num_conns'):
-  #  node += "Conns:"+str(v['num_conns'])+"\\l"
-  #node +="\""
-  #node +=",group=\""+v['external-ip'].replace(".","") + "\""
-  #node += "]"
   print node
 
   for con in v.get('to',[]):
@@ -130,27 +86,20 @@ def write_node(k,v):
     weight = str(1000 - (((w - 150) * (1000 - 0)) / (1000 -150 )) + 0)
 
     length = str(float(w)/1500)
-    #weight = "1000" #str(300/float(con['weight']))
-    #weight = str((100/float(con['weight'])))
-    #weight = str(-1 * (200-100000/int(con['weight'])))
     if float(weight) < 0 :
       weight= "1"
 
-    #sys.stderr.write(weight + ":"+ length +" %s -> " %k + str(con) + "\n")
-    edge = "  "+k+ " -> " +con['name'] + " [label="+label + " weight="+weight #+ " minlen="+length
+    edge = "  "+k+ " -> " +con['name'] + " [label="+label + " weight="+weight 
     if con.get('bidirectional',False):
       edge += ",dir=both"
     edge += "]"
     print edge
-
-def decode_input(FILE):
-  return json.load(FILE)
-nodes = decode_input(sys.stdin)
-nodes = delete_unused_nodes(nodes)
-write_digraph(nodes)
-try: 
-  end = time()
-  msg = '%s.graph.anon_build_time %d %d\r\n' % (g_path,((end-begin)*1000),end)
-  s.send(msg)
-  s.close()
-except Exception as e: pass
+if __name__ == "__main__":
+  nodes = delete_unused_nodes(json.load(sys.stdin))
+  write_digraph(nodes)
+  try: 
+    end = time()
+    msg = '%s.graph.anon_build_time %d %d\r\n' % (g_path,((end-begin)*1000),end)
+    s.send(msg)
+    s.close()
+  except Exception as e: pass
diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
index e6a67e2c..4383b356 100755
--- a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
+++ b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
@@ -1,8 +1,10 @@
 #!/usr/bin/python
 # -*- coding: utf8 -*-
-from BackwardsReader import BackwardsReader
+from tinc_stats.BackwardsReader import BackwardsReader
+from tinc_stats.Graph import generate_stats,delete_unused_nodes,merge_edges,get_node_avg_weight
+from tinc_stats.Supernodes import check_all_the_super
+from tinc_stats.Availability import get_node_availability
 import sys,json
-from find_super import check_all_the_super
 try:
   from time import time
   import socket
@@ -100,65 +102,6 @@ def generate_stats(nodes):
     v['availability'] = get_node_availability(k,jlines)
     sys.stderr.write( "%s -> %f\n" %(k ,v['availability']))
 
-def get_node_avg_weight(conns):
-  """ calculates the average weight for the given connections """
-  if not conns:
-    sys.syderr.write("get_node_avg_weight: connection parameter empty")
-    return 9001
-  else:
-    return sum([float(c['weight']) for c in conns])/len(conns)
-def get_node_availability(name,jlines):
-  """ calculates the node availability by reading the generated dump file
-  adding together the uptime of the node and returning the time
-	parms:
-          name - node name
-          jlines - list of already parsed dictionaries node archive
-  """
-  begin = last = current = 0
-  uptime = 0
-  #sys.stderr.write ( "Getting Node availability of %s\n" % name)
-  for stat in jlines:
-    if not stat['nodes']:
-      continue
-    ts = stat['timestamp']
-    if not begin:
-      begin = last = ts
-    current = ts
-    if stat['nodes'].get(name,{}).get('to',[]):
-      uptime += current - last
-    else:
-      pass
-      #sys.stderr.write("%s offline at timestamp %f\n" %(name,current))
-    last = ts
-  all_the_time = last - begin
-  try:
-    return uptime/ all_the_time
-  except:
-    return 1
-
-def delete_unused_nodes(nodes):
-  new_nodes = {}
-  for k,v in nodes.iteritems():
-    if v['external-ip'] == "(null)":
-      continue
-    if v.get('to',[]):
-      new_nodes[k] = v
-  for k,v in new_nodes.iteritems():
-    if not [ i for i in v['to'] if i['name'] in new_nodes]:
-      #del(new_nodes[k])
-      del(k)
-  return new_nodes
-
-def merge_edges(nodes):
-  """ merge back and forth edges into one
-  DESTRUCTS the current structure by deleting "connections" in the nodes
-  """
-  for k,v in nodes.iteritems():
-    for con in v.get('to',[]):
-      for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])):
-        if k == secon['name']:
-          del (nodes[con['name']]['to'][i])
-          con['bidirectional'] = True
 
 
 def write_node(k,v):
diff --git a/retiolum/scripts/adv_graphgen/sanitize.sh b/retiolum/scripts/adv_graphgen/sanitize.sh
index c8071dc3..7d556c68 100755
--- a/retiolum/scripts/adv_graphgen/sanitize.sh
+++ b/retiolum/scripts/adv_graphgen/sanitize.sh
@@ -11,7 +11,7 @@ TYPE2=png
 OPENER=/bin/true
 DOTFILE=`mktemp --suffix=san`
 trap 'rm $DOTFILE' INT TERM KILL EXIT
-sudo -E python tinc_stats2json |\
+sudo -E python tinc_stats/Log2JSON.py |\
     python parse_tinc_stats.py > $DOTFILE
 
 
diff --git a/retiolum/scripts/adv_graphgen/find_legacy_hosts.py b/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py
index 52388b6d..a1ef13f1 100755
--- a/retiolum/scripts/adv_graphgen/find_legacy_hosts.py
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py
@@ -9,8 +9,9 @@ DUMP_FILE = "/krebs/db/availability"
 def get_all_nodes():
   import os
   return os.listdir("/etc/tinc/retiolum/hosts")
+
 def generate_stats():
-  """ Generates some statistics of the network and nodes
+  """ Generates availability statistics of the network and nodes
   """
   import json
   jlines = []
@@ -35,7 +36,6 @@ def get_node_availability(name,jlines):
   """
   begin = last = current = 0
   uptime = 0
-  #sys.stderr.write ( "Getting Node availability of %s\n" % name)
   for stat in jlines:
     if not stat['nodes']:
       continue
@@ -47,7 +47,6 @@ def get_node_availability(name,jlines):
       uptime += current - last
     else:
       pass
-      #sys.stderr.write("%s offline at timestamp %f\n" %(name,current))
     last = ts
   all_the_time = last - begin
   try:
@@ -55,5 +54,5 @@ def get_node_availability(name,jlines):
   except:
     return 1
 
-
-generate_stats()
+if __name__ == "__main__":
+  generate_stats()
diff --git a/retiolum/scripts/adv_graphgen/BackwardsReader.py b/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py
index 6bdbf43c..6bdbf43c 100644
--- a/retiolum/scripts/adv_graphgen/BackwardsReader.py
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py
diff --git a/retiolum/scripts/adv_graphgen/nodes_geoip.py b/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py
index 7f12d213..ba3f6738 100755
--- a/retiolum/scripts/adv_graphgen/nodes_geoip.py
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py
@@ -1,25 +1,12 @@
 #!/usr/bin/python3
 # -*- coding: utf8 -*-
 import sys,json
-
-def delete_unused_nodes(nodes):
-  new_nodes = {}
-  for k,v in nodes.iteritems():
-    if v['external-ip'] == "(null)":
-      continue
-    if v.get('to',[]):
-      new_nodes[k] = v
-  for k,v in new_nodes.iteritems():
-    if not [ i for i in v['to'] if i['name'] in new_nodes]:
-      #del(new_nodes[k])
-      del(k)
-  return new_nodes
+from Graph import delete_unused_nodes
 
 if __name__ == "__main__":
   from pygeoip import GeoIP
   gi = GeoIP("GeoLiteCity.dat")
   for node,data in delete_unused_nodes(json.load(sys.stdin)).items():
-    
     try:
       print ("%s in %s"%(node,gi.record_by_addr(data["external-ip"])["city"]))
     except:
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py b/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py
new file mode 100644
index 00000000..26db3030
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py
@@ -0,0 +1,44 @@
+#!/usr/bin/python
+
+def generate_stats(nodes):
+  """ Generates some statistics of the network and nodes
+  """
+  for k,v in nodes.iteritems():
+    conns = v.get('to',[])
+    for c in conns: #sanitize weights
+      if float(c['weight']) > 9000: c['weight'] = str(9001)
+      elif float(c['weight']) < 0: c['weight'] = str(0)
+    v['num_conns'] = len(conns)
+    v['avg_weight'] = get_node_avg_weight(conns)
+
+def get_node_avg_weight(conns):
+  """ calculates the average weight for the given connections """
+  if not conns:
+    sys.syderr.write("get_node_avg_weight: connection parameter empty")
+    return 9001
+  else:
+    return sum([float(c['weight']) for c in conns])/len(conns)
+
+def delete_unused_nodes(nodes):
+  """ Deletes all the nodes which are currently not connected to the network"""
+  new_nodes = {}
+  for k,v in nodes.iteritems():
+    if v['external-ip'] == "(null)":
+      continue
+    if v.get('to',[]):
+      new_nodes[k] = v
+  for k,v in new_nodes.iteritems():
+    if not [ i for i in v['to'] if i['name'] in new_nodes]:
+      del(k)
+  return new_nodes
+
+def merge_edges(nodes):
+  """ merge back and forth edges into one
+  DESTRUCTS the current structure by deleting "connections" in the nodes
+  """
+  for k,v in nodes.iteritems():
+    for con in v.get('to',[]):
+      for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])):
+        if k == secon['name']:
+          del (nodes[con['name']]['to'][i])
+          con['bidirectional'] = True
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats2json b/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py
index 644cbc63..644cbc63 100755
--- a/retiolum/scripts/adv_graphgen/tinc_stats2json
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py
diff --git a/retiolum/scripts/adv_graphgen/find-all-nodes.py b/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py
index ae0fae8f..ae0fae8f 100644
--- a/retiolum/scripts/adv_graphgen/find-all-nodes.py
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py b/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py
new file mode 100644
index 00000000..62f541d1
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py
@@ -0,0 +1,6 @@
+import Availability
+import BackwardsReader
+import Log2JSON
+import Supernodes
+import Geo
+import Graph