From 2860ed57fd203716c5cf4b2c5e02e79b5d3dce58 Mon Sep 17 00:00:00 2001 From: euer Date: Thu, 29 Nov 2012 16:39:44 +0100 Subject: add find_super script to find valid supernodes fix stderr issues --- retiolum/scripts/adv_graphgen/find_super.py | 50 +++++++++++++++++++++++ retiolum/scripts/adv_graphgen/parse_tinc_anon.py | 4 +- retiolum/scripts/adv_graphgen/parse_tinc_stats.py | 12 +++--- 3 files changed, 59 insertions(+), 7 deletions(-) create mode 100644 retiolum/scripts/adv_graphgen/find_super.py (limited to 'retiolum/scripts/adv_graphgen') diff --git a/retiolum/scripts/adv_graphgen/find_super.py b/retiolum/scripts/adv_graphgen/find_super.py new file mode 100644 index 00000000..df01734e --- /dev/null +++ b/retiolum/scripts/adv_graphgen/find_super.py @@ -0,0 +1,50 @@ +#!/usr/bin/python + +def find_super(path="/etc/tinc/retiolum/hosts"): + import os + import re + + needle_addr = re.compile("Address\s*=\s*(.*)") + needle_port = re.compile("Port\s*=\s*(.*)") + for f in os.listdir(path): + with open(path+"/"+f) as of: + addrs = [] + port = "655" + + for line in of.readlines(): + + addr_found = needle_addr.match(line) + if addr_found: + addrs.append(addr_found.group(1)) + + port_found = needle_port.match(line) + if port_found: + port = port_found.group(1) + + if addrs : yield (f ,[(addr ,int(port)) for addr in addrs]) + +def check_super(path="/etc/tinc/retiolum/hosts"): + from socket import socket,AF_INET,SOCK_STREAM + for host,addrs in find_super(path): + valid_addrs = [] + for addr in addrs: + try: + s = socket(AF_INET,SOCK_STREAM) + s.settimeout(3) + s.connect(addr) + #print("success connecting %s:%d"%(addr)) + s.settimeout(None) + s.close() + valid_addrs.append(addr) + except Exception as e: + pass + #print("cannot connect to %s:%d"%(addr)) + if valid_addrs: yield (host,valid_addrs) + + +if __name__ == "__main__": + """ + usage + """ + for host,addrs in check_super(): + print host,addrs diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_anon.py b/retiolum/scripts/adv_graphgen/parse_tinc_anon.py index e0bea913..21c36e0f 100755 --- a/retiolum/scripts/adv_graphgen/parse_tinc_anon.py +++ b/retiolum/scripts/adv_graphgen/parse_tinc_anon.py @@ -15,7 +15,7 @@ try: sys.stderr.write("connecting to %s:%d"%(host,port)) s.connect((host,port)) except Exception as e: - print >>sys.stderr, "Cannot connect to graphite: " + str(e) + sys.stderr.write( "Cannot connect to graphite: " + str(e)) """ TODO: Refactoring needed to pull the edges out of the node structures again, it should be easier to handle both structures""" DUMP_FILE = "/krebs/db/availability" @@ -151,4 +151,4 @@ try: msg = '%s.graph.anon_build_time %d %d\r\n' % (g_path,((end-begin)*1000),end) s.send(msg) s.close() -except Exception as e: print >>sys.stderr, e +except Exception as e: pass diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py index 16f4f795..8371ad3d 100755 --- a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py +++ b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py @@ -2,6 +2,7 @@ # -*- coding: utf8 -*- from BackwardsReader import BackwardsReader import sys,json +from find_super import check_super try: from time import time import socket @@ -16,7 +17,10 @@ try: except Exception as e: sys.stderr.write("Cannot connect to graphite: %s\n" % str(e)) -supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ] +supernodes= [ ] +#supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ] +for supernode,addr in check_super(): + supernodes.append(supernode) """ TODO: Refactoring needed to pull the edges out of the node structures again, it should be easier to handle both structures""" DUMP_FILE = "/krebs/db/availability" @@ -53,8 +57,7 @@ def write_stat_node(nodes): try: msg = '%s.num_nodes %d %d\r\n' %(g_path,num_nodes,begin) s.send(msg) - #print >>sys.stderr, msg - except Exception as e: print sys.stderr,e + except Exception as e: pass #except: pass for k,v in nodes.iteritems(): num_conns+= len(v['to']) @@ -82,8 +85,7 @@ def generate_stats(nodes): jlines.append(jline) lines_to_use -=1 - except Exception,e: - sys.stderr.write(str(e)) + except Exception,e: sys.stderr.write(str(e)) for k,v in nodes.iteritems(): conns = v.get('to',[]) for c in conns: #sanitize weights -- cgit v1.2.3 From 52ba57cf7473a8f480728eaa8c3616952bb7335a Mon Sep 17 00:00:00 2001 From: euer Date: Fri, 7 Dec 2012 22:53:12 +0100 Subject: tinc_stats.py -> tinc_stats2json tinc_stats2json is now able to parse new tincctl format as well as legacy syslog format --- retiolum/scripts/adv_graphgen/anonytize.sh | 2 +- retiolum/scripts/adv_graphgen/sanitize.sh | 2 +- retiolum/scripts/adv_graphgen/tinc_stats.py | 83 ----------------------------- 3 files changed, 2 insertions(+), 85 deletions(-) delete mode 100755 retiolum/scripts/adv_graphgen/tinc_stats.py (limited to 'retiolum/scripts/adv_graphgen') diff --git a/retiolum/scripts/adv_graphgen/anonytize.sh b/retiolum/scripts/adv_graphgen/anonytize.sh index d49793cb..dec6e456 100755 --- a/retiolum/scripts/adv_graphgen/anonytize.sh +++ b/retiolum/scripts/adv_graphgen/anonytize.sh @@ -11,7 +11,7 @@ TYPE2=png OPENER=/bin/true DOTFILE=`mktemp` trap 'rm $DOTFILE' INT TERM -sudo LOG_FILE=$LOG_FILE python tinc_stats.py |\ +sudo LOG_FILE=$LOG_FILE python ../../tinc_stats2json |\ python parse_tinc_anon.py> $DOTFILE diff --git a/retiolum/scripts/adv_graphgen/sanitize.sh b/retiolum/scripts/adv_graphgen/sanitize.sh index c46662f3..78d74ce6 100755 --- a/retiolum/scripts/adv_graphgen/sanitize.sh +++ b/retiolum/scripts/adv_graphgen/sanitize.sh @@ -11,7 +11,7 @@ TYPE2=png OPENER=/bin/true DOTFILE=`mktemp` trap 'rm $DOTFILE' INT TERM -sudo LOG_FILE=$LOG_FILE python tinc_stats.py |\ +sudo LOG_FILE=$LOG_FILE python ../../tinc_stats2json |\ python parse_tinc_stats.py > $DOTFILE diff --git a/retiolum/scripts/adv_graphgen/tinc_stats.py b/retiolum/scripts/adv_graphgen/tinc_stats.py deleted file mode 100755 index d0d47aff..00000000 --- a/retiolum/scripts/adv_graphgen/tinc_stats.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/python -from BackwardsReader import BackwardsReader -import os -import re -import sys -import json - - -TINC_NETWORK = os.environ.get("TINC_NETWORK","retiolum") -os.environ["LOG_FILE"] -SYSLOG_FILE = os.environ.get("LOG_FILE","/var/log/everything.log") - - -# Tags and Delimiters -TINC_TAG="tinc.%s" % TINC_NETWORK -BEGIN_NODES = "Nodes:" -END_NODES = "End of nodes." -BEGIN_SUBNET = "Subnet list:" -END_SUBNET = "End of subnet list" -BEGIN_EDGES = "Edges:" -END_EDGES = "End of edges." - -def get_tinc_block(log_file): - """ returns an iterateable block from the given log file (syslog) """ - tinc_block = [] - in_block = False - bf = BackwardsReader(log_file) - BOL = re.compile(".*tinc.retiolum\[[0-9]+\]: ") - while True: - line = bf.readline() - if not line: - raise Exception("end of file at log file? This should not happen!") - line = BOL.sub('',line).strip() - - if END_SUBNET in line: - in_block = True - - if not in_block: - continue - - tinc_block.append(line) - - if BEGIN_NODES in line: - break - return reversed(tinc_block) - -def parse_input(log_data): - nodes={} - for line in log_data: - if BEGIN_NODES in line : - nodes={} - for line in log_data: - if END_NODES in line : - break - l = line.replace('\n','').split() #TODO unhack me - nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] } - if BEGIN_SUBNET in line : - for line in log_data: - if END_SUBNET in line : - break - l = line.replace('\n','').split() - if not nodes[l[2]].get('internal-ip',False): - nodes[l[2]]['internal-ip'] = [] - nodes[l[2]]['internal-ip'].append(l[0].split('#')[0]) - if BEGIN_EDGES in line : - edges = {} - for line in log_data: - if END_EDGES in line : - break - l = line.replace('\n','').split() - - if not nodes[l[0]].has_key('to') : - nodes[l[0]]['to'] = [] - nodes[l[0]]['to'].append( - {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] }) - return nodes - - -if __name__ == '__main__': - import subprocess,time - subprocess.call(["pkill","-SIGUSR2", "tincd"]) - time.sleep(1) - print json.dumps(parse_input((get_tinc_block(SYSLOG_FILE)))) -- cgit v1.2.3 From c971ce27a09d4aa5e740fda3664c8a8181de3233 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 2 Jan 2013 18:34:41 +0000 Subject: fix adv_graphgen, github_listener --- retiolum/scripts/adv_graphgen/all_the_graphs.sh | 6 +- retiolum/scripts/adv_graphgen/tinc_stats2json | 124 ++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 2 deletions(-) create mode 100755 retiolum/scripts/adv_graphgen/tinc_stats2json (limited to 'retiolum/scripts/adv_graphgen') diff --git a/retiolum/scripts/adv_graphgen/all_the_graphs.sh b/retiolum/scripts/adv_graphgen/all_the_graphs.sh index 5533c722..8aa0ab21 100755 --- a/retiolum/scripts/adv_graphgen/all_the_graphs.sh +++ b/retiolum/scripts/adv_graphgen/all_the_graphs.sh @@ -5,10 +5,12 @@ cd $(dirname $(readlink -f $0)) PATH=$PATH:../../../util/bin/ export LOG_FILE=/var/log/retiolum.log + EXTERNAL_FOLDER=/var/www/euer.krebsco.de/graphs/retiolum + INTERNAL_FOLDER=/var/www/euer/graphs/retiolum begin=`timer` export GRAPHITE_HOST="no_omo" - (./anonytize.sh /srv/http/pub/graphs/retiolum/ && echo "`date` anonytize done" >> /tmp/build_graph)& - (./sanitize.sh /srv/http/priv/graphs/retiolum/ && echo "`date` sanitize done" >> /tmp/build_graph)& + (./anonytize.sh $EXTERNAL_FOLDER && echo "`date` anonytize done" >> /tmp/build_graph)& + (./sanitize.sh $INTERNAL_FOLDER && echo "`date` sanitize done" >> /tmp/build_graph)& # wait graphitec "retiolum.graph.buildtime" "$(timer $begin)" >> /tmp/build_graph echo "`date` end all graphs" >> /tmp/build_graph diff --git a/retiolum/scripts/adv_graphgen/tinc_stats2json b/retiolum/scripts/adv_graphgen/tinc_stats2json new file mode 100755 index 00000000..ede19b26 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_stats2json @@ -0,0 +1,124 @@ +#!/usr/bin/python +import subprocess +import os +import re +import sys +import json + + + +TINC_NETWORK =os.environ.get("TINC_NETWORK","retiolum") + +# is_legacy is the parameter which defines if the tinc config files are handled old fashioned (parse from syslog), +# or if the new and hip tincctl should be used +is_legacy= os.environ.get("TINC_LEGACY",False) +SYSLOG_FILE = os.environ.get("LOG_FILE","/var/log/everything.log") + + +# Tags and Delimiters +TINC_TAG="tinc.%s" % TINC_NETWORK +BEGIN_NODES = "Nodes:" +END_NODES = "End of nodes." +BEGIN_SUBNET = "Subnet list:" +END_SUBNET = "End of subnet list" +BEGIN_EDGES = "Edges:" +END_EDGES = "End of edges." + +def get_tinc_block(log_file): + """ returns an iterateable block from the given log file (syslog) + This function became obsolete with the introduction of tincctl + """ + from BackwardsReader import BackwardsReader + tinc_block = [] + in_block = False + bf = BackwardsReader(log_file) + BOL = re.compile(".*tinc.%s\[[0-9]+\]: " % TINC_NETWORK) + while True: + line = bf.readline() + if not line: + raise Exception("end of file at log file? This should not happen!") + line = BOL.sub('',line).strip() + + if END_SUBNET in line: + in_block = True + + if not in_block: + continue + + tinc_block.append(line) + + if BEGIN_NODES in line: + break + return reversed(tinc_block) + +def parse_new_input(): + nodes = {} + pnodes = subprocess.Popen(["tincctl","-n",TINC_NETWORK,"dump","reachable","nodes"], stdout=subprocess.PIPE).communicate()[0] + #pnodes = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","reachable","nodes"]) + for line in pnodes.split('\n'): + if not line: continue + l = line.split() + nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] } + psubnets = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","subnets"]) + for line in psubnets.split('\n'): + if not line: continue + l = line.split() + try: + if not nodes[l[2]].get('internal-ip',False): + nodes[l[2]]['internal-ip'] = [] + nodes[l[2]]['internal-ip'].append(l[0].split('#')[0]) + except KeyError: + pass # node does not exist (presumably) + pedges = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","edges"]) + for line in pedges.split('\n'): + if not line: continue + l = line.split() + try: + if not nodes[l[0]].has_key('to') : + nodes[l[0]]['to'] = [] + nodes[l[0]]['to'].append( + {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] }) + except KeyError: + pass #node does not exist + return nodes + +def parse_input(log_data): + nodes={} + for line in log_data: + if BEGIN_NODES in line : + nodes={} + for line in log_data: + if END_NODES in line : + break + l = line.replace('\n','').split() #TODO unhack me + nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] } + if BEGIN_SUBNET in line : + for line in log_data: + if END_SUBNET in line : + break + l = line.replace('\n','').split() + if not nodes[l[2]].get('internal-ip',False): + nodes[l[2]]['internal-ip'] = [] + nodes[l[2]]['internal-ip'].append(l[0].split('#')[0]) + if BEGIN_EDGES in line : + edges = {} + for line in log_data: + if END_EDGES in line : + break + l = line.replace('\n','').split() + if not nodes[l[0]].has_key('to') : + nodes[l[0]]['to'] = [] + nodes[l[0]]['to'].append( + {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] }) + return nodes + + +if __name__ == '__main__': + import subprocess,time + if is_legacy: + subprocess.call(["pkill","-SIGUSR2", "tincd"]) + time.sleep(1) + print json.dumps(parse_input((get_tinc_block(SYSLOG_FILE)))) + else: + print json.dumps(parse_new_input()) + -- cgit v1.2.3 From 414f96a2431e1a61412c2c5b4533fbcbb53499f6 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 6 Jan 2013 02:01:17 +0000 Subject: fix graph generation --- retiolum/scripts/adv_graphgen/all_the_graphs.sh | 3 ++- retiolum/scripts/adv_graphgen/anonytize.sh | 2 +- retiolum/scripts/adv_graphgen/sanitize.sh | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) (limited to 'retiolum/scripts/adv_graphgen') diff --git a/retiolum/scripts/adv_graphgen/all_the_graphs.sh b/retiolum/scripts/adv_graphgen/all_the_graphs.sh index 8aa0ab21..d3ce8f86 100755 --- a/retiolum/scripts/adv_graphgen/all_the_graphs.sh +++ b/retiolum/scripts/adv_graphgen/all_the_graphs.sh @@ -4,7 +4,8 @@ echo "`date` begin all graphs" >> /tmp/build_graph cd $(dirname $(readlink -f $0)) PATH=$PATH:../../../util/bin/ - export LOG_FILE=/var/log/retiolum.log + export LOG_FILE=/var/log/syslog + export TINC_LEGACY=true EXTERNAL_FOLDER=/var/www/euer.krebsco.de/graphs/retiolum INTERNAL_FOLDER=/var/www/euer/graphs/retiolum begin=`timer` diff --git a/retiolum/scripts/adv_graphgen/anonytize.sh b/retiolum/scripts/adv_graphgen/anonytize.sh index dec6e456..b31f4dbb 100755 --- a/retiolum/scripts/adv_graphgen/anonytize.sh +++ b/retiolum/scripts/adv_graphgen/anonytize.sh @@ -11,7 +11,7 @@ TYPE2=png OPENER=/bin/true DOTFILE=`mktemp` trap 'rm $DOTFILE' INT TERM -sudo LOG_FILE=$LOG_FILE python ../../tinc_stats2json |\ +sudo -E python tinc_stats2json |\ python parse_tinc_anon.py> $DOTFILE diff --git a/retiolum/scripts/adv_graphgen/sanitize.sh b/retiolum/scripts/adv_graphgen/sanitize.sh index 78d74ce6..45d29a22 100755 --- a/retiolum/scripts/adv_graphgen/sanitize.sh +++ b/retiolum/scripts/adv_graphgen/sanitize.sh @@ -11,7 +11,7 @@ TYPE2=png OPENER=/bin/true DOTFILE=`mktemp` trap 'rm $DOTFILE' INT TERM -sudo LOG_FILE=$LOG_FILE python ../../tinc_stats2json |\ +sudo -E python tinc_stats2json |\ python parse_tinc_stats.py > $DOTFILE -- cgit v1.2.3 From 29d8a4e5061d66d0dfeda28a9012992c12a83ccd Mon Sep 17 00:00:00 2001 From: makefu Date: Sun, 6 Jan 2013 04:40:20 +0000 Subject: adv_graphgen: add visual warning for nodes with only one connection as this means no redundancy is configured in case the other connected node dies away --- retiolum/scripts/adv_graphgen/parse_tinc_stats.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'retiolum/scripts/adv_graphgen') diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py index 8371ad3d..03a20214 100755 --- a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py +++ b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py @@ -18,12 +18,12 @@ except Exception as e: sys.stderr.write("Cannot connect to graphite: %s\n" % str(e)) supernodes= [ ] -#supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ] for supernode,addr in check_super(): supernodes.append(supernode) """ TODO: Refactoring needed to pull the edges out of the node structures again, it should be easier to handle both structures""" DUMP_FILE = "/krebs/db/availability" + def write_digraph(nodes): """ writes the complete digraph in dot format @@ -173,6 +173,9 @@ def write_node(k,v): node +="\"" if k in supernodes: node += ",fillcolor=steelblue1" + # warning if node only has one connection + elif v['num_conns'] == 1: + node += ",fillcolor=red" #node +=",group=\""+v['external-ip'].replace(".","")+"\"" node += "]" print node -- cgit v1.2.3 From bba5171e63c88ae4b0850adc0a6bf49a55a19662 Mon Sep 17 00:00:00 2001 From: makefu Date: Sun, 6 Jan 2013 04:43:53 +0000 Subject: adv_graphgen: no redundancy > being supernode it is visually more important to see that a node has no redundancy that to see this node is a potential supernode --- retiolum/scripts/adv_graphgen/parse_tinc_stats.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'retiolum/scripts/adv_graphgen') diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py index 03a20214..76a3ffcd 100755 --- a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py +++ b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py @@ -171,11 +171,12 @@ def write_node(k,v): for addr in v.get('internal-ip',['¯\\\\(°_o)/¯']): node += "internal:"+addr+"\\l" node +="\"" - if k in supernodes: - node += ",fillcolor=steelblue1" + # warning if node only has one connection - elif v['num_conns'] == 1: + if v['num_conns'] == 1: node += ",fillcolor=red" + elif k in supernodes: + node += ",fillcolor=steelblue1" #node +=",group=\""+v['external-ip'].replace(".","")+"\"" node += "]" print node -- cgit v1.2.3