From fe1f4dec340c121e43d0fca6d1139db694b2c48b Mon Sep 17 00:00:00 2001 From: root Date: Mon, 24 Jun 2013 17:24:49 +0200 Subject: refactor tinc_stats into a single python module --- retiolum/scripts/adv_graphgen/BackwardsReader.py | 35 ----- retiolum/scripts/adv_graphgen/anonytize.sh | 2 +- retiolum/scripts/adv_graphgen/find-all-nodes.py | 59 -------- retiolum/scripts/adv_graphgen/find_legacy_hosts.py | 59 -------- retiolum/scripts/adv_graphgen/find_super.py | 59 -------- retiolum/scripts/adv_graphgen/nodes_geoip.py | 26 ---- retiolum/scripts/adv_graphgen/parse_tinc_anon.py | 79 ++-------- retiolum/scripts/adv_graphgen/parse_tinc_stats.py | 65 +------- retiolum/scripts/adv_graphgen/sanitize.sh | 2 +- .../adv_graphgen/tinc_stats/Availability.py | 58 +++++++ .../adv_graphgen/tinc_stats/BackwardsReader.py | 35 +++++ retiolum/scripts/adv_graphgen/tinc_stats/Geo.py | 13 ++ retiolum/scripts/adv_graphgen/tinc_stats/Graph.py | 44 ++++++ .../scripts/adv_graphgen/tinc_stats/Log2JSON.py | 166 +++++++++++++++++++++ .../scripts/adv_graphgen/tinc_stats/Supernodes.py | 59 ++++++++ .../scripts/adv_graphgen/tinc_stats/__init__.py | 6 + retiolum/scripts/adv_graphgen/tinc_stats2json | 166 --------------------- 17 files changed, 401 insertions(+), 532 deletions(-) delete mode 100644 retiolum/scripts/adv_graphgen/BackwardsReader.py delete mode 100644 retiolum/scripts/adv_graphgen/find-all-nodes.py delete mode 100755 retiolum/scripts/adv_graphgen/find_legacy_hosts.py delete mode 100644 retiolum/scripts/adv_graphgen/find_super.py delete mode 100755 retiolum/scripts/adv_graphgen/nodes_geoip.py create mode 100755 retiolum/scripts/adv_graphgen/tinc_stats/Availability.py create mode 100644 retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py create mode 100755 retiolum/scripts/adv_graphgen/tinc_stats/Geo.py create mode 100644 retiolum/scripts/adv_graphgen/tinc_stats/Graph.py create mode 100755 retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py create mode 100644 retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py create mode 100644 retiolum/scripts/adv_graphgen/tinc_stats/__init__.py delete mode 100755 retiolum/scripts/adv_graphgen/tinc_stats2json (limited to 'retiolum/scripts') diff --git a/retiolum/scripts/adv_graphgen/BackwardsReader.py b/retiolum/scripts/adv_graphgen/BackwardsReader.py deleted file mode 100644 index 6bdbf43c..00000000 --- a/retiolum/scripts/adv_graphgen/BackwardsReader.py +++ /dev/null @@ -1,35 +0,0 @@ -import sys -import os -import string - -class BackwardsReader: - """ Stripped and stolen from : http://code.activestate.com/recipes/120686-read-a-text-file-backwards/ """ - def readline(self): - while len(self.data) == 1 and ((self.blkcount * self.blksize) < self.size): - self.blkcount = self.blkcount + 1 - line = self.data[0] - try: - self.f.seek(-self.blksize * self.blkcount, 2) - self.data = string.split(self.f.read(self.blksize) + line, '\n') - except IOError: - self.f.seek(0) - self.data = string.split(self.f.read(self.size - (self.blksize * (self.blkcount-1))) + line, '\n') - - if len(self.data) == 0: - return "" - - line = self.data[-1] - self.data = self.data[:-1] - return line + '\n' - - def __init__(self, file, blksize=4096): - """initialize the internal structures""" - self.size = os.stat(file)[6] - self.blksize = blksize - self.blkcount = 1 - self.f = open(file, 'rb') - if self.size > self.blksize: - self.f.seek(-self.blksize * self.blkcount, 2) - self.data = string.split(self.f.read(self.blksize), '\n') - if not self.data[-1]: - self.data = self.data[:-1] diff --git a/retiolum/scripts/adv_graphgen/anonytize.sh b/retiolum/scripts/adv_graphgen/anonytize.sh index 4aad6993..4ed5c8db 100755 --- a/retiolum/scripts/adv_graphgen/anonytize.sh +++ b/retiolum/scripts/adv_graphgen/anonytize.sh @@ -11,7 +11,7 @@ TYPE2=png OPENER=/bin/true DOTFILE=`mktemp --suffix=anon` trap 'rm $DOTFILE' INT TERM KILL EXIT -sudo -E python tinc_stats2json |\ +sudo -E python tinc_stats/Log2JSON.py |\ python parse_tinc_anon.py> $DOTFILE diff --git a/retiolum/scripts/adv_graphgen/find-all-nodes.py b/retiolum/scripts/adv_graphgen/find-all-nodes.py deleted file mode 100644 index ae0fae8f..00000000 --- a/retiolum/scripts/adv_graphgen/find-all-nodes.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/python - -def find_potential_super(path="/etc/tinc/retiolum/hosts"): - import os - import re - - needle_addr = re.compile("Address\s*=\s*(.*)") - needle_port = re.compile("Port\s*=\s*(.*)") - for f in os.listdir(path): - with open(path+"/"+f) as of: - addrs = [] - port = "655" - - for line in of.readlines(): - - addr_found = needle_addr.match(line) - if addr_found: - addrs.append(addr_found.group(1)) - - port_found = needle_port.match(line) - if port_found: - port = port_found.group(1) - - if addrs : yield (f ,[(addr ,int(port)) for addr in addrs]) - -def try_connect(addr): - try: - from socket import socket,AF_INET,SOCK_STREAM - s = socket(AF_INET,SOCK_STREAM) - s.settimeout(2) - s.connect(addr) - s.settimeout(None) - s.close() - return addr - except Exception as e: - pass - #return () - -def check_one_super(ha): - host,addrs = ha - valid_addrs = [] - for addr in addrs: - ret = try_connect(addr) - if ret: valid_addrs.append(ret) - if valid_addrs: return (host,valid_addrs) - -def check_all_the_super(path="/etc/tinc/retiolum/hosts"): - from multiprocessing import Pool - p = Pool(20) - return filter(None,p.map(check_one_super,find_potential_super(path))) - - - -if __name__ == "__main__": - """ - usage - """ - for host,addrs in check_all_the_super(): - print host,addrs diff --git a/retiolum/scripts/adv_graphgen/find_legacy_hosts.py b/retiolum/scripts/adv_graphgen/find_legacy_hosts.py deleted file mode 100755 index 52388b6d..00000000 --- a/retiolum/scripts/adv_graphgen/find_legacy_hosts.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf8 -*- - -import sys,json -""" TODO: Refactoring needed to pull the edges out of the node structures again, -it should be easier to handle both structures""" -DUMP_FILE = "/krebs/db/availability" - -def get_all_nodes(): - import os - return os.listdir("/etc/tinc/retiolum/hosts") -def generate_stats(): - """ Generates some statistics of the network and nodes - """ - import json - jlines = [] - try: - f = open(DUMP_FILE,'r') - for line in f: - jlines.append(json.loads(line)) - f.close() - except Exception,e: - pass - all_nodes = {} - for k in get_all_nodes(): - all_nodes[k] = get_node_availability(k,jlines) - print ( json.dumps(all_nodes)) - -def get_node_availability(name,jlines): - """ calculates the node availability by reading the generated dump file - adding together the uptime of the node and returning the time - parms: - name - node name - jlines - list of already parsed dictionaries node archive - """ - begin = last = current = 0 - uptime = 0 - #sys.stderr.write ( "Getting Node availability of %s\n" % name) - for stat in jlines: - if not stat['nodes']: - continue - ts = stat['timestamp'] - if not begin: - begin = last = ts - current = ts - if stat['nodes'].get(name,{}).get('to',[]): - uptime += current - last - else: - pass - #sys.stderr.write("%s offline at timestamp %f\n" %(name,current)) - last = ts - all_the_time = last - begin - try: - return uptime/ all_the_time - except: - return 1 - - -generate_stats() diff --git a/retiolum/scripts/adv_graphgen/find_super.py b/retiolum/scripts/adv_graphgen/find_super.py deleted file mode 100644 index ae0fae8f..00000000 --- a/retiolum/scripts/adv_graphgen/find_super.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/python - -def find_potential_super(path="/etc/tinc/retiolum/hosts"): - import os - import re - - needle_addr = re.compile("Address\s*=\s*(.*)") - needle_port = re.compile("Port\s*=\s*(.*)") - for f in os.listdir(path): - with open(path+"/"+f) as of: - addrs = [] - port = "655" - - for line in of.readlines(): - - addr_found = needle_addr.match(line) - if addr_found: - addrs.append(addr_found.group(1)) - - port_found = needle_port.match(line) - if port_found: - port = port_found.group(1) - - if addrs : yield (f ,[(addr ,int(port)) for addr in addrs]) - -def try_connect(addr): - try: - from socket import socket,AF_INET,SOCK_STREAM - s = socket(AF_INET,SOCK_STREAM) - s.settimeout(2) - s.connect(addr) - s.settimeout(None) - s.close() - return addr - except Exception as e: - pass - #return () - -def check_one_super(ha): - host,addrs = ha - valid_addrs = [] - for addr in addrs: - ret = try_connect(addr) - if ret: valid_addrs.append(ret) - if valid_addrs: return (host,valid_addrs) - -def check_all_the_super(path="/etc/tinc/retiolum/hosts"): - from multiprocessing import Pool - p = Pool(20) - return filter(None,p.map(check_one_super,find_potential_super(path))) - - - -if __name__ == "__main__": - """ - usage - """ - for host,addrs in check_all_the_super(): - print host,addrs diff --git a/retiolum/scripts/adv_graphgen/nodes_geoip.py b/retiolum/scripts/adv_graphgen/nodes_geoip.py deleted file mode 100755 index 7f12d213..00000000 --- a/retiolum/scripts/adv_graphgen/nodes_geoip.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/python3 -# -*- coding: utf8 -*- -import sys,json - -def delete_unused_nodes(nodes): - new_nodes = {} - for k,v in nodes.iteritems(): - if v['external-ip'] == "(null)": - continue - if v.get('to',[]): - new_nodes[k] = v - for k,v in new_nodes.iteritems(): - if not [ i for i in v['to'] if i['name'] in new_nodes]: - #del(new_nodes[k]) - del(k) - return new_nodes - -if __name__ == "__main__": - from pygeoip import GeoIP - gi = GeoIP("GeoLiteCity.dat") - for node,data in delete_unused_nodes(json.load(sys.stdin)).items(): - - try: - print ("%s in %s"%(node,gi.record_by_addr(data["external-ip"])["city"])) - except: - print ("%s myself"%node) diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_anon.py b/retiolum/scripts/adv_graphgen/parse_tinc_anon.py index 82ee2f2e..9041fb35 100755 --- a/retiolum/scripts/adv_graphgen/parse_tinc_anon.py +++ b/retiolum/scripts/adv_graphgen/parse_tinc_anon.py @@ -1,6 +1,7 @@ #!/usr/bin/python # -*- coding: utf8 -*- -from BackwardsReader import BackwardsReader +from tinc_stats.BackwardsReader import BackwardsReader +from tinc_stats.Graph import generate_stats,delete_unused_nodes,merge_edges import sys,json #supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ] try: @@ -19,6 +20,7 @@ except Exception as e: """ TODO: Refactoring needed to pull the edges out of the node structures again, it should be easier to handle both structures""" DUMP_FILE = "/krebs/db/availability" + def write_digraph(nodes): """ writes the complete digraph in dot format @@ -37,6 +39,7 @@ def write_digraph(nodes): print ('}') def anon_nodes(nodes): + #anonymizes all nodes i = "0" newnodes = {} for k,v in nodes.iteritems(): @@ -66,46 +69,6 @@ def write_stat_node(nodes): node_text += "]" print(node_text) -def generate_stats(nodes): - """ Generates some statistics of the network and nodes - """ - for k,v in nodes.iteritems(): - conns = v.get('to',[]) - for c in conns: #sanitize weights - if float(c['weight']) > 9000: c['weight'] = str(9001) - elif float(c['weight']) < 0: c['weight'] = str(0) - v['num_conns'] = len(conns) - v['avg_weight'] = get_node_avg_weight(conns) -def get_node_avg_weight(conns): - """ calculates the average weight for the given connections """ - if not conns: - sys.syderr.write("get_node_avg_weight: connection parameter empty") - return 9001 - else: - return sum([float(c['weight']) for c in conns])/len(conns) - -def delete_unused_nodes(nodes): - new_nodes = {} - for k,v in nodes.iteritems(): - if v['external-ip'] == "(null)": - continue - if v.get('to',[]): - new_nodes[k] = v - for k,v in new_nodes.iteritems(): - if not [ i for i in v['to'] if i['name'] in new_nodes]: - #del(new_nodes[k]) - del(k) - return new_nodes -def merge_edges(nodes): - """ merge back and forth edges into one - DESTRUCTS the current structure by deleting "connections" in the nodes - """ - for k,v in nodes.iteritems(): - for con in v.get('to',[]): - for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])): - if k == secon['name']: - del (nodes[con['name']]['to'][i]) - con['bidirectional'] = True def write_node(k,v): @@ -115,13 +78,6 @@ def write_node(k,v): """ node = " "+k #+"[label=\"" - #node += k+"\\l" - #node += "avg weight: %.2f\\l" % v['avg_weight'] - #if v.has_key('num_conns'): - # node += "Conns:"+str(v['num_conns'])+"\\l" - #node +="\"" - #node +=",group=\""+v['external-ip'].replace(".","") + "\"" - #node += "]" print node for con in v.get('to',[]): @@ -130,27 +86,20 @@ def write_node(k,v): weight = str(1000 - (((w - 150) * (1000 - 0)) / (1000 -150 )) + 0) length = str(float(w)/1500) - #weight = "1000" #str(300/float(con['weight'])) - #weight = str((100/float(con['weight']))) - #weight = str(-1 * (200-100000/int(con['weight']))) if float(weight) < 0 : weight= "1" - #sys.stderr.write(weight + ":"+ length +" %s -> " %k + str(con) + "\n") - edge = " "+k+ " -> " +con['name'] + " [label="+label + " weight="+weight #+ " minlen="+length + edge = " "+k+ " -> " +con['name'] + " [label="+label + " weight="+weight if con.get('bidirectional',False): edge += ",dir=both" edge += "]" print edge - -def decode_input(FILE): - return json.load(FILE) -nodes = decode_input(sys.stdin) -nodes = delete_unused_nodes(nodes) -write_digraph(nodes) -try: - end = time() - msg = '%s.graph.anon_build_time %d %d\r\n' % (g_path,((end-begin)*1000),end) - s.send(msg) - s.close() -except Exception as e: pass +if __name__ == "__main__": + nodes = delete_unused_nodes(json.load(sys.stdin)) + write_digraph(nodes) + try: + end = time() + msg = '%s.graph.anon_build_time %d %d\r\n' % (g_path,((end-begin)*1000),end) + s.send(msg) + s.close() + except Exception as e: pass diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py index e6a67e2c..4383b356 100755 --- a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py +++ b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py @@ -1,8 +1,10 @@ #!/usr/bin/python # -*- coding: utf8 -*- -from BackwardsReader import BackwardsReader +from tinc_stats.BackwardsReader import BackwardsReader +from tinc_stats.Graph import generate_stats,delete_unused_nodes,merge_edges,get_node_avg_weight +from tinc_stats.Supernodes import check_all_the_super +from tinc_stats.Availability import get_node_availability import sys,json -from find_super import check_all_the_super try: from time import time import socket @@ -100,65 +102,6 @@ def generate_stats(nodes): v['availability'] = get_node_availability(k,jlines) sys.stderr.write( "%s -> %f\n" %(k ,v['availability'])) -def get_node_avg_weight(conns): - """ calculates the average weight for the given connections """ - if not conns: - sys.syderr.write("get_node_avg_weight: connection parameter empty") - return 9001 - else: - return sum([float(c['weight']) for c in conns])/len(conns) -def get_node_availability(name,jlines): - """ calculates the node availability by reading the generated dump file - adding together the uptime of the node and returning the time - parms: - name - node name - jlines - list of already parsed dictionaries node archive - """ - begin = last = current = 0 - uptime = 0 - #sys.stderr.write ( "Getting Node availability of %s\n" % name) - for stat in jlines: - if not stat['nodes']: - continue - ts = stat['timestamp'] - if not begin: - begin = last = ts - current = ts - if stat['nodes'].get(name,{}).get('to',[]): - uptime += current - last - else: - pass - #sys.stderr.write("%s offline at timestamp %f\n" %(name,current)) - last = ts - all_the_time = last - begin - try: - return uptime/ all_the_time - except: - return 1 - -def delete_unused_nodes(nodes): - new_nodes = {} - for k,v in nodes.iteritems(): - if v['external-ip'] == "(null)": - continue - if v.get('to',[]): - new_nodes[k] = v - for k,v in new_nodes.iteritems(): - if not [ i for i in v['to'] if i['name'] in new_nodes]: - #del(new_nodes[k]) - del(k) - return new_nodes - -def merge_edges(nodes): - """ merge back and forth edges into one - DESTRUCTS the current structure by deleting "connections" in the nodes - """ - for k,v in nodes.iteritems(): - for con in v.get('to',[]): - for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])): - if k == secon['name']: - del (nodes[con['name']]['to'][i]) - con['bidirectional'] = True def write_node(k,v): diff --git a/retiolum/scripts/adv_graphgen/sanitize.sh b/retiolum/scripts/adv_graphgen/sanitize.sh index c8071dc3..7d556c68 100755 --- a/retiolum/scripts/adv_graphgen/sanitize.sh +++ b/retiolum/scripts/adv_graphgen/sanitize.sh @@ -11,7 +11,7 @@ TYPE2=png OPENER=/bin/true DOTFILE=`mktemp --suffix=san` trap 'rm $DOTFILE' INT TERM KILL EXIT -sudo -E python tinc_stats2json |\ +sudo -E python tinc_stats/Log2JSON.py |\ python parse_tinc_stats.py > $DOTFILE diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py b/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py new file mode 100755 index 00000000..a1ef13f1 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py @@ -0,0 +1,58 @@ +#!/usr/bin/python +# -*- coding: utf8 -*- + +import sys,json +""" TODO: Refactoring needed to pull the edges out of the node structures again, +it should be easier to handle both structures""" +DUMP_FILE = "/krebs/db/availability" + +def get_all_nodes(): + import os + return os.listdir("/etc/tinc/retiolum/hosts") + +def generate_stats(): + """ Generates availability statistics of the network and nodes + """ + import json + jlines = [] + try: + f = open(DUMP_FILE,'r') + for line in f: + jlines.append(json.loads(line)) + f.close() + except Exception,e: + pass + all_nodes = {} + for k in get_all_nodes(): + all_nodes[k] = get_node_availability(k,jlines) + print ( json.dumps(all_nodes)) + +def get_node_availability(name,jlines): + """ calculates the node availability by reading the generated dump file + adding together the uptime of the node and returning the time + parms: + name - node name + jlines - list of already parsed dictionaries node archive + """ + begin = last = current = 0 + uptime = 0 + for stat in jlines: + if not stat['nodes']: + continue + ts = stat['timestamp'] + if not begin: + begin = last = ts + current = ts + if stat['nodes'].get(name,{}).get('to',[]): + uptime += current - last + else: + pass + last = ts + all_the_time = last - begin + try: + return uptime/ all_the_time + except: + return 1 + +if __name__ == "__main__": + generate_stats() diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py b/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py new file mode 100644 index 00000000..6bdbf43c --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py @@ -0,0 +1,35 @@ +import sys +import os +import string + +class BackwardsReader: + """ Stripped and stolen from : http://code.activestate.com/recipes/120686-read-a-text-file-backwards/ """ + def readline(self): + while len(self.data) == 1 and ((self.blkcount * self.blksize) < self.size): + self.blkcount = self.blkcount + 1 + line = self.data[0] + try: + self.f.seek(-self.blksize * self.blkcount, 2) + self.data = string.split(self.f.read(self.blksize) + line, '\n') + except IOError: + self.f.seek(0) + self.data = string.split(self.f.read(self.size - (self.blksize * (self.blkcount-1))) + line, '\n') + + if len(self.data) == 0: + return "" + + line = self.data[-1] + self.data = self.data[:-1] + return line + '\n' + + def __init__(self, file, blksize=4096): + """initialize the internal structures""" + self.size = os.stat(file)[6] + self.blksize = blksize + self.blkcount = 1 + self.f = open(file, 'rb') + if self.size > self.blksize: + self.f.seek(-self.blksize * self.blkcount, 2) + self.data = string.split(self.f.read(self.blksize), '\n') + if not self.data[-1]: + self.data = self.data[:-1] diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py b/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py new file mode 100755 index 00000000..ba3f6738 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py @@ -0,0 +1,13 @@ +#!/usr/bin/python3 +# -*- coding: utf8 -*- +import sys,json +from Graph import delete_unused_nodes + +if __name__ == "__main__": + from pygeoip import GeoIP + gi = GeoIP("GeoLiteCity.dat") + for node,data in delete_unused_nodes(json.load(sys.stdin)).items(): + try: + print ("%s in %s"%(node,gi.record_by_addr(data["external-ip"])["city"])) + except: + print ("%s myself"%node) diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py b/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py new file mode 100644 index 00000000..26db3030 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py @@ -0,0 +1,44 @@ +#!/usr/bin/python + +def generate_stats(nodes): + """ Generates some statistics of the network and nodes + """ + for k,v in nodes.iteritems(): + conns = v.get('to',[]) + for c in conns: #sanitize weights + if float(c['weight']) > 9000: c['weight'] = str(9001) + elif float(c['weight']) < 0: c['weight'] = str(0) + v['num_conns'] = len(conns) + v['avg_weight'] = get_node_avg_weight(conns) + +def get_node_avg_weight(conns): + """ calculates the average weight for the given connections """ + if not conns: + sys.syderr.write("get_node_avg_weight: connection parameter empty") + return 9001 + else: + return sum([float(c['weight']) for c in conns])/len(conns) + +def delete_unused_nodes(nodes): + """ Deletes all the nodes which are currently not connected to the network""" + new_nodes = {} + for k,v in nodes.iteritems(): + if v['external-ip'] == "(null)": + continue + if v.get('to',[]): + new_nodes[k] = v + for k,v in new_nodes.iteritems(): + if not [ i for i in v['to'] if i['name'] in new_nodes]: + del(k) + return new_nodes + +def merge_edges(nodes): + """ merge back and forth edges into one + DESTRUCTS the current structure by deleting "connections" in the nodes + """ + for k,v in nodes.iteritems(): + for con in v.get('to',[]): + for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])): + if k == secon['name']: + del (nodes[con['name']]['to'][i]) + con['bidirectional'] = True diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py b/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py new file mode 100755 index 00000000..644cbc63 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py @@ -0,0 +1,166 @@ +#!/usr/bin/python +import subprocess +import os +import re +import sys +import json + + +TINC_NETWORK =os.environ.get("TINC_NETWORK","retiolum") + +# is_legacy is the parameter which defines if the tinc config files are handled old fashioned (parse from syslog), +# or if the new and hip tincctl should be used + + +# Tags and Delimiters +TINC_TAG="tinc.%s" % TINC_NETWORK +BEGIN_NODES = "Nodes:" +END_NODES = "End of nodes." +BEGIN_SUBNET = "Subnet list:" +END_SUBNET = "End of subnet list" +BEGIN_EDGES = "Edges:" +END_EDGES = "End of edges." +def usage(): + from sys import argv,exit + print("""usage: %s +This tool dumps all tinc node informations as json + +ENVIRONMENT VARIABLES: + TINC_NETWORK The tinc network to dump + (default: retiolum) + LOG_FILE If legacy tinc is used, defines the log file where tinc stats are dumped in + (default: /var/log/everything.log) +""" % argv[0]) + exit(1) +def debug(func): + from functools import wraps + @wraps(func) + def with_debug(*args,**kwargs): + print( func.__name__ + " (args: %s | kwargs %s)"% (args,kwargs)) + return func(*args,**kwargs) + return with_debug + + +def get_tinc_log_file(): + # TODO parse logfile from somewhere + return os.environ.get("LOG_FILE","/var/log/everything.log") + + +def parse_tinc_stats(): + import subprocess + from time import sleep + from distutils.spawn import find_executable as which + #newest tinc + if which("tinc"): + return parse_new_input("tinc") + #new tinc + elif which("tincctl"): + return parse_new_input("tincctl") + #old tinc + elif which("tincd"): + # TODO refactor me + subprocess.call(["pkill","-SIGUSR2", "tincd"]) + sleep(1) + return parse_input(get_tinc_block(get_tinc_log_file())) + #no tinc + else: + raise Exception("no tinc executable found!") + +#@debug +def get_tinc_block(log_file): + """ returns an iterateable block from the given log file (syslog) + This function became obsolete with the introduction of tincctl + """ + from BackwardsReader import BackwardsReader + tinc_block = [] + in_block = False + bf = BackwardsReader(log_file) + BOL = re.compile(".*tinc.%s\[[0-9]+\]: " % TINC_NETWORK) + while True: + line = bf.readline() + if not line: + raise Exception("end of file at log file? This should not happen!") + line = BOL.sub('',line).strip() + + if END_SUBNET in line: + in_block = True + + if not in_block: + continue + tinc_block.append(line) + + if BEGIN_NODES in line: + break + return reversed(tinc_block) + +def parse_new_input(tinc_bin): + nodes = {} + pnodes = subprocess.Popen([tinc_bin,"-n",TINC_NETWORK,"dump","reachable","nodes"], stdout=subprocess.PIPE).communicate()[0] + #pnodes = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","reachable","nodes"]) + for line in pnodes.split('\n'): + if not line: continue + l = line.split() + nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] } + psubnets = subprocess.check_output([tinc_bin,"-n",TINC_NETWORK,"dump","subnets"]) + for line in psubnets.split('\n'): + if not line: continue + l = line.split() + try: + if not nodes[l[2]].get('internal-ip',False): + nodes[l[2]]['internal-ip'] = [] + nodes[l[2]]['internal-ip'].append(l[0].split('#')[0]) + except KeyError: + pass # node does not exist (presumably) + pedges = subprocess.check_output([tinc_bin,"-n",TINC_NETWORK,"dump","edges"]) + for line in pedges.split('\n'): + if not line: continue + l = line.split() + try: + if not nodes[l[0]].has_key('to') : + nodes[l[0]]['to'] = [] + nodes[l[0]]['to'].append( + {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] }) + except KeyError: + pass #node does not exist + return nodes + +#@debug +def parse_input(log_data): + nodes={} + for line in log_data: + if BEGIN_NODES in line : + nodes={} + for line in log_data: + if END_NODES in line : + break + l = line.replace('\n','').split() #TODO unhack me + nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] } + if BEGIN_SUBNET in line : + for line in log_data: + if END_SUBNET in line : + break + l = line.replace('\n','').split() + if not nodes[l[2]].get('internal-ip',False): + nodes[l[2]]['internal-ip'] = [] + nodes[l[2]]['internal-ip'].append(l[0].split('#')[0]) + if BEGIN_EDGES in line : + edges = {} + for line in log_data: + if END_EDGES in line : + break + l = line.replace('\n','').split() + if not nodes[l[0]].has_key('to') : + nodes[l[0]]['to'] = [] + nodes[l[0]]['to'].append( + {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] }) + return nodes + + +if __name__ == '__main__': + # TODO refactor me + from sys import argv + if len(argv) > 1: + usage() + else: + print json.dumps(parse_tinc_stats()) + diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py b/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py new file mode 100644 index 00000000..ae0fae8f --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py @@ -0,0 +1,59 @@ +#!/usr/bin/python + +def find_potential_super(path="/etc/tinc/retiolum/hosts"): + import os + import re + + needle_addr = re.compile("Address\s*=\s*(.*)") + needle_port = re.compile("Port\s*=\s*(.*)") + for f in os.listdir(path): + with open(path+"/"+f) as of: + addrs = [] + port = "655" + + for line in of.readlines(): + + addr_found = needle_addr.match(line) + if addr_found: + addrs.append(addr_found.group(1)) + + port_found = needle_port.match(line) + if port_found: + port = port_found.group(1) + + if addrs : yield (f ,[(addr ,int(port)) for addr in addrs]) + +def try_connect(addr): + try: + from socket import socket,AF_INET,SOCK_STREAM + s = socket(AF_INET,SOCK_STREAM) + s.settimeout(2) + s.connect(addr) + s.settimeout(None) + s.close() + return addr + except Exception as e: + pass + #return () + +def check_one_super(ha): + host,addrs = ha + valid_addrs = [] + for addr in addrs: + ret = try_connect(addr) + if ret: valid_addrs.append(ret) + if valid_addrs: return (host,valid_addrs) + +def check_all_the_super(path="/etc/tinc/retiolum/hosts"): + from multiprocessing import Pool + p = Pool(20) + return filter(None,p.map(check_one_super,find_potential_super(path))) + + + +if __name__ == "__main__": + """ + usage + """ + for host,addrs in check_all_the_super(): + print host,addrs diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py b/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py new file mode 100644 index 00000000..62f541d1 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py @@ -0,0 +1,6 @@ +import Availability +import BackwardsReader +import Log2JSON +import Supernodes +import Geo +import Graph diff --git a/retiolum/scripts/adv_graphgen/tinc_stats2json b/retiolum/scripts/adv_graphgen/tinc_stats2json deleted file mode 100755 index 644cbc63..00000000 --- a/retiolum/scripts/adv_graphgen/tinc_stats2json +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/python -import subprocess -import os -import re -import sys -import json - - -TINC_NETWORK =os.environ.get("TINC_NETWORK","retiolum") - -# is_legacy is the parameter which defines if the tinc config files are handled old fashioned (parse from syslog), -# or if the new and hip tincctl should be used - - -# Tags and Delimiters -TINC_TAG="tinc.%s" % TINC_NETWORK -BEGIN_NODES = "Nodes:" -END_NODES = "End of nodes." -BEGIN_SUBNET = "Subnet list:" -END_SUBNET = "End of subnet list" -BEGIN_EDGES = "Edges:" -END_EDGES = "End of edges." -def usage(): - from sys import argv,exit - print("""usage: %s -This tool dumps all tinc node informations as json - -ENVIRONMENT VARIABLES: - TINC_NETWORK The tinc network to dump - (default: retiolum) - LOG_FILE If legacy tinc is used, defines the log file where tinc stats are dumped in - (default: /var/log/everything.log) -""" % argv[0]) - exit(1) -def debug(func): - from functools import wraps - @wraps(func) - def with_debug(*args,**kwargs): - print( func.__name__ + " (args: %s | kwargs %s)"% (args,kwargs)) - return func(*args,**kwargs) - return with_debug - - -def get_tinc_log_file(): - # TODO parse logfile from somewhere - return os.environ.get("LOG_FILE","/var/log/everything.log") - - -def parse_tinc_stats(): - import subprocess - from time import sleep - from distutils.spawn import find_executable as which - #newest tinc - if which("tinc"): - return parse_new_input("tinc") - #new tinc - elif which("tincctl"): - return parse_new_input("tincctl") - #old tinc - elif which("tincd"): - # TODO refactor me - subprocess.call(["pkill","-SIGUSR2", "tincd"]) - sleep(1) - return parse_input(get_tinc_block(get_tinc_log_file())) - #no tinc - else: - raise Exception("no tinc executable found!") - -#@debug -def get_tinc_block(log_file): - """ returns an iterateable block from the given log file (syslog) - This function became obsolete with the introduction of tincctl - """ - from BackwardsReader import BackwardsReader - tinc_block = [] - in_block = False - bf = BackwardsReader(log_file) - BOL = re.compile(".*tinc.%s\[[0-9]+\]: " % TINC_NETWORK) - while True: - line = bf.readline() - if not line: - raise Exception("end of file at log file? This should not happen!") - line = BOL.sub('',line).strip() - - if END_SUBNET in line: - in_block = True - - if not in_block: - continue - tinc_block.append(line) - - if BEGIN_NODES in line: - break - return reversed(tinc_block) - -def parse_new_input(tinc_bin): - nodes = {} - pnodes = subprocess.Popen([tinc_bin,"-n",TINC_NETWORK,"dump","reachable","nodes"], stdout=subprocess.PIPE).communicate()[0] - #pnodes = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","reachable","nodes"]) - for line in pnodes.split('\n'): - if not line: continue - l = line.split() - nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] } - psubnets = subprocess.check_output([tinc_bin,"-n",TINC_NETWORK,"dump","subnets"]) - for line in psubnets.split('\n'): - if not line: continue - l = line.split() - try: - if not nodes[l[2]].get('internal-ip',False): - nodes[l[2]]['internal-ip'] = [] - nodes[l[2]]['internal-ip'].append(l[0].split('#')[0]) - except KeyError: - pass # node does not exist (presumably) - pedges = subprocess.check_output([tinc_bin,"-n",TINC_NETWORK,"dump","edges"]) - for line in pedges.split('\n'): - if not line: continue - l = line.split() - try: - if not nodes[l[0]].has_key('to') : - nodes[l[0]]['to'] = [] - nodes[l[0]]['to'].append( - {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] }) - except KeyError: - pass #node does not exist - return nodes - -#@debug -def parse_input(log_data): - nodes={} - for line in log_data: - if BEGIN_NODES in line : - nodes={} - for line in log_data: - if END_NODES in line : - break - l = line.replace('\n','').split() #TODO unhack me - nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] } - if BEGIN_SUBNET in line : - for line in log_data: - if END_SUBNET in line : - break - l = line.replace('\n','').split() - if not nodes[l[2]].get('internal-ip',False): - nodes[l[2]]['internal-ip'] = [] - nodes[l[2]]['internal-ip'].append(l[0].split('#')[0]) - if BEGIN_EDGES in line : - edges = {} - for line in log_data: - if END_EDGES in line : - break - l = line.replace('\n','').split() - if not nodes[l[0]].has_key('to') : - nodes[l[0]]['to'] = [] - nodes[l[0]]['to'].append( - {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] }) - return nodes - - -if __name__ == '__main__': - # TODO refactor me - from sys import argv - if len(argv) > 1: - usage() - else: - print json.dumps(parse_tinc_stats()) - -- cgit v1.2.3