diff options
12 files changed, 75 insertions, 206 deletions
diff --git a/retiolum/scripts/adv_graphgen/anonytize.sh b/retiolum/scripts/adv_graphgen/anonytize.sh index 4aad6993..4ed5c8db 100755 --- a/retiolum/scripts/adv_graphgen/anonytize.sh +++ b/retiolum/scripts/adv_graphgen/anonytize.sh @@ -11,7 +11,7 @@ TYPE2=png OPENER=/bin/true DOTFILE=`mktemp --suffix=anon` trap 'rm $DOTFILE' INT TERM KILL EXIT -sudo -E python tinc_stats2json |\ +sudo -E python tinc_stats/Log2JSON.py |\ python parse_tinc_anon.py> $DOTFILE diff --git a/retiolum/scripts/adv_graphgen/find_super.py b/retiolum/scripts/adv_graphgen/find_super.py deleted file mode 100644 index ae0fae8f..00000000 --- a/retiolum/scripts/adv_graphgen/find_super.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/python - -def find_potential_super(path="/etc/tinc/retiolum/hosts"): - import os - import re - - needle_addr = re.compile("Address\s*=\s*(.*)") - needle_port = re.compile("Port\s*=\s*(.*)") - for f in os.listdir(path): - with open(path+"/"+f) as of: - addrs = [] - port = "655" - - for line in of.readlines(): - - addr_found = needle_addr.match(line) - if addr_found: - addrs.append(addr_found.group(1)) - - port_found = needle_port.match(line) - if port_found: - port = port_found.group(1) - - if addrs : yield (f ,[(addr ,int(port)) for addr in addrs]) - -def try_connect(addr): - try: - from socket import socket,AF_INET,SOCK_STREAM - s = socket(AF_INET,SOCK_STREAM) - s.settimeout(2) - s.connect(addr) - s.settimeout(None) - s.close() - return addr - except Exception as e: - pass - #return () - -def check_one_super(ha): - host,addrs = ha - valid_addrs = [] - for addr in addrs: - ret = try_connect(addr) - if ret: valid_addrs.append(ret) - if valid_addrs: return (host,valid_addrs) - -def check_all_the_super(path="/etc/tinc/retiolum/hosts"): - from multiprocessing import Pool - p = Pool(20) - return filter(None,p.map(check_one_super,find_potential_super(path))) - - - -if __name__ == "__main__": - """ - usage - """ - for host,addrs in check_all_the_super(): - print host,addrs diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_anon.py b/retiolum/scripts/adv_graphgen/parse_tinc_anon.py index 82ee2f2e..9041fb35 100755 --- a/retiolum/scripts/adv_graphgen/parse_tinc_anon.py +++ b/retiolum/scripts/adv_graphgen/parse_tinc_anon.py @@ -1,6 +1,7 @@ #!/usr/bin/python # -*- coding: utf8 -*- -from BackwardsReader import BackwardsReader +from tinc_stats.BackwardsReader import BackwardsReader +from tinc_stats.Graph import generate_stats,delete_unused_nodes,merge_edges import sys,json #supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ] try: @@ -19,6 +20,7 @@ except Exception as e: """ TODO: Refactoring needed to pull the edges out of the node structures again, it should be easier to handle both structures""" DUMP_FILE = "/krebs/db/availability" + def write_digraph(nodes): """ writes the complete digraph in dot format @@ -37,6 +39,7 @@ def write_digraph(nodes): print ('}') def anon_nodes(nodes): + #anonymizes all nodes i = "0" newnodes = {} for k,v in nodes.iteritems(): @@ -66,46 +69,6 @@ def write_stat_node(nodes): node_text += "]" print(node_text) -def generate_stats(nodes): - """ Generates some statistics of the network and nodes - """ - for k,v in nodes.iteritems(): - conns = v.get('to',[]) - for c in conns: #sanitize weights - if float(c['weight']) > 9000: c['weight'] = str(9001) - elif float(c['weight']) < 0: c['weight'] = str(0) - v['num_conns'] = len(conns) - v['avg_weight'] = get_node_avg_weight(conns) -def get_node_avg_weight(conns): - """ calculates the average weight for the given connections """ - if not conns: - sys.syderr.write("get_node_avg_weight: connection parameter empty") - return 9001 - else: - return sum([float(c['weight']) for c in conns])/len(conns) - -def delete_unused_nodes(nodes): - new_nodes = {} - for k,v in nodes.iteritems(): - if v['external-ip'] == "(null)": - continue - if v.get('to',[]): - new_nodes[k] = v - for k,v in new_nodes.iteritems(): - if not [ i for i in v['to'] if i['name'] in new_nodes]: - #del(new_nodes[k]) - del(k) - return new_nodes -def merge_edges(nodes): - """ merge back and forth edges into one - DESTRUCTS the current structure by deleting "connections" in the nodes - """ - for k,v in nodes.iteritems(): - for con in v.get('to',[]): - for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])): - if k == secon['name']: - del (nodes[con['name']]['to'][i]) - con['bidirectional'] = True def write_node(k,v): @@ -115,13 +78,6 @@ def write_node(k,v): """ node = " "+k #+"[label=\"" - #node += k+"\\l" - #node += "avg weight: %.2f\\l" % v['avg_weight'] - #if v.has_key('num_conns'): - # node += "Conns:"+str(v['num_conns'])+"\\l" - #node +="\"" - #node +=",group=\""+v['external-ip'].replace(".","") + "\"" - #node += "]" print node for con in v.get('to',[]): @@ -130,27 +86,20 @@ def write_node(k,v): weight = str(1000 - (((w - 150) * (1000 - 0)) / (1000 -150 )) + 0) length = str(float(w)/1500) - #weight = "1000" #str(300/float(con['weight'])) - #weight = str((100/float(con['weight']))) - #weight = str(-1 * (200-100000/int(con['weight']))) if float(weight) < 0 : weight= "1" - #sys.stderr.write(weight + ":"+ length +" %s -> " %k + str(con) + "\n") - edge = " "+k+ " -> " +con['name'] + " [label="+label + " weight="+weight #+ " minlen="+length + edge = " "+k+ " -> " +con['name'] + " [label="+label + " weight="+weight if con.get('bidirectional',False): edge += ",dir=both" edge += "]" print edge - -def decode_input(FILE): - return json.load(FILE) -nodes = decode_input(sys.stdin) -nodes = delete_unused_nodes(nodes) -write_digraph(nodes) -try: - end = time() - msg = '%s.graph.anon_build_time %d %d\r\n' % (g_path,((end-begin)*1000),end) - s.send(msg) - s.close() -except Exception as e: pass +if __name__ == "__main__": + nodes = delete_unused_nodes(json.load(sys.stdin)) + write_digraph(nodes) + try: + end = time() + msg = '%s.graph.anon_build_time %d %d\r\n' % (g_path,((end-begin)*1000),end) + s.send(msg) + s.close() + except Exception as e: pass diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py index e6a67e2c..4383b356 100755 --- a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py +++ b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py @@ -1,8 +1,10 @@ #!/usr/bin/python # -*- coding: utf8 -*- -from BackwardsReader import BackwardsReader +from tinc_stats.BackwardsReader import BackwardsReader +from tinc_stats.Graph import generate_stats,delete_unused_nodes,merge_edges,get_node_avg_weight +from tinc_stats.Supernodes import check_all_the_super +from tinc_stats.Availability import get_node_availability import sys,json -from find_super import check_all_the_super try: from time import time import socket @@ -100,65 +102,6 @@ def generate_stats(nodes): v['availability'] = get_node_availability(k,jlines) sys.stderr.write( "%s -> %f\n" %(k ,v['availability'])) -def get_node_avg_weight(conns): - """ calculates the average weight for the given connections """ - if not conns: - sys.syderr.write("get_node_avg_weight: connection parameter empty") - return 9001 - else: - return sum([float(c['weight']) for c in conns])/len(conns) -def get_node_availability(name,jlines): - """ calculates the node availability by reading the generated dump file - adding together the uptime of the node and returning the time - parms: - name - node name - jlines - list of already parsed dictionaries node archive - """ - begin = last = current = 0 - uptime = 0 - #sys.stderr.write ( "Getting Node availability of %s\n" % name) - for stat in jlines: - if not stat['nodes']: - continue - ts = stat['timestamp'] - if not begin: - begin = last = ts - current = ts - if stat['nodes'].get(name,{}).get('to',[]): - uptime += current - last - else: - pass - #sys.stderr.write("%s offline at timestamp %f\n" %(name,current)) - last = ts - all_the_time = last - begin - try: - return uptime/ all_the_time - except: - return 1 - -def delete_unused_nodes(nodes): - new_nodes = {} - for k,v in nodes.iteritems(): - if v['external-ip'] == "(null)": - continue - if v.get('to',[]): - new_nodes[k] = v - for k,v in new_nodes.iteritems(): - if not [ i for i in v['to'] if i['name'] in new_nodes]: - #del(new_nodes[k]) - del(k) - return new_nodes - -def merge_edges(nodes): - """ merge back and forth edges into one - DESTRUCTS the current structure by deleting "connections" in the nodes - """ - for k,v in nodes.iteritems(): - for con in v.get('to',[]): - for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])): - if k == secon['name']: - del (nodes[con['name']]['to'][i]) - con['bidirectional'] = True def write_node(k,v): diff --git a/retiolum/scripts/adv_graphgen/sanitize.sh b/retiolum/scripts/adv_graphgen/sanitize.sh index c8071dc3..7d556c68 100755 --- a/retiolum/scripts/adv_graphgen/sanitize.sh +++ b/retiolum/scripts/adv_graphgen/sanitize.sh @@ -11,7 +11,7 @@ TYPE2=png OPENER=/bin/true DOTFILE=`mktemp --suffix=san` trap 'rm $DOTFILE' INT TERM KILL EXIT -sudo -E python tinc_stats2json |\ +sudo -E python tinc_stats/Log2JSON.py |\ python parse_tinc_stats.py > $DOTFILE diff --git a/retiolum/scripts/adv_graphgen/find_legacy_hosts.py b/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py index 52388b6d..a1ef13f1 100755 --- a/retiolum/scripts/adv_graphgen/find_legacy_hosts.py +++ b/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py @@ -9,8 +9,9 @@ DUMP_FILE = "/krebs/db/availability" def get_all_nodes(): import os return os.listdir("/etc/tinc/retiolum/hosts") + def generate_stats(): - """ Generates some statistics of the network and nodes + """ Generates availability statistics of the network and nodes """ import json jlines = [] @@ -35,7 +36,6 @@ def get_node_availability(name,jlines): """ begin = last = current = 0 uptime = 0 - #sys.stderr.write ( "Getting Node availability of %s\n" % name) for stat in jlines: if not stat['nodes']: continue @@ -47,7 +47,6 @@ def get_node_availability(name,jlines): uptime += current - last else: pass - #sys.stderr.write("%s offline at timestamp %f\n" %(name,current)) last = ts all_the_time = last - begin try: @@ -55,5 +54,5 @@ def get_node_availability(name,jlines): except: return 1 - -generate_stats() +if __name__ == "__main__": + generate_stats() diff --git a/retiolum/scripts/adv_graphgen/BackwardsReader.py b/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py index 6bdbf43c..6bdbf43c 100644 --- a/retiolum/scripts/adv_graphgen/BackwardsReader.py +++ b/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py diff --git a/retiolum/scripts/adv_graphgen/nodes_geoip.py b/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py index 7f12d213..ba3f6738 100755 --- a/retiolum/scripts/adv_graphgen/nodes_geoip.py +++ b/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py @@ -1,25 +1,12 @@ #!/usr/bin/python3 # -*- coding: utf8 -*- import sys,json - -def delete_unused_nodes(nodes): - new_nodes = {} - for k,v in nodes.iteritems(): - if v['external-ip'] == "(null)": - continue - if v.get('to',[]): - new_nodes[k] = v - for k,v in new_nodes.iteritems(): - if not [ i for i in v['to'] if i['name'] in new_nodes]: - #del(new_nodes[k]) - del(k) - return new_nodes +from Graph import delete_unused_nodes if __name__ == "__main__": from pygeoip import GeoIP gi = GeoIP("GeoLiteCity.dat") for node,data in delete_unused_nodes(json.load(sys.stdin)).items(): - try: print ("%s in %s"%(node,gi.record_by_addr(data["external-ip"])["city"])) except: diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py b/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py new file mode 100644 index 00000000..26db3030 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py @@ -0,0 +1,44 @@ +#!/usr/bin/python + +def generate_stats(nodes): + """ Generates some statistics of the network and nodes + """ + for k,v in nodes.iteritems(): + conns = v.get('to',[]) + for c in conns: #sanitize weights + if float(c['weight']) > 9000: c['weight'] = str(9001) + elif float(c['weight']) < 0: c['weight'] = str(0) + v['num_conns'] = len(conns) + v['avg_weight'] = get_node_avg_weight(conns) + +def get_node_avg_weight(conns): + """ calculates the average weight for the given connections """ + if not conns: + sys.syderr.write("get_node_avg_weight: connection parameter empty") + return 9001 + else: + return sum([float(c['weight']) for c in conns])/len(conns) + +def delete_unused_nodes(nodes): + """ Deletes all the nodes which are currently not connected to the network""" + new_nodes = {} + for k,v in nodes.iteritems(): + if v['external-ip'] == "(null)": + continue + if v.get('to',[]): + new_nodes[k] = v + for k,v in new_nodes.iteritems(): + if not [ i for i in v['to'] if i['name'] in new_nodes]: + del(k) + return new_nodes + +def merge_edges(nodes): + """ merge back and forth edges into one + DESTRUCTS the current structure by deleting "connections" in the nodes + """ + for k,v in nodes.iteritems(): + for con in v.get('to',[]): + for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])): + if k == secon['name']: + del (nodes[con['name']]['to'][i]) + con['bidirectional'] = True diff --git a/retiolum/scripts/adv_graphgen/tinc_stats2json b/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py index 644cbc63..644cbc63 100755 --- a/retiolum/scripts/adv_graphgen/tinc_stats2json +++ b/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py diff --git a/retiolum/scripts/adv_graphgen/find-all-nodes.py b/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py index ae0fae8f..ae0fae8f 100644 --- a/retiolum/scripts/adv_graphgen/find-all-nodes.py +++ b/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py b/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py new file mode 100644 index 00000000..62f541d1 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py @@ -0,0 +1,6 @@ +import Availability +import BackwardsReader +import Log2JSON +import Supernodes +import Geo +import Graph |