summaryrefslogtreecommitdiffstats
path: root/retiolum
diff options
context:
space:
mode:
Diffstat (limited to 'retiolum')
-rwxr-xr-xretiolum/bin/my-ip2
l---------retiolum/bin/tinc_stats1
-rw-r--r--retiolum/hosts/euer (renamed from retiolum/hosts/EUcancER)0
-rw-r--r--retiolum/scripts/adv_graphgen/BackwardsReader.py35
-rwxr-xr-xretiolum/scripts/adv_graphgen/parse.py110
-rwxr-xr-xretiolum/scripts/adv_graphgen/parse_tinc_stats.py175
-rwxr-xr-xretiolum/scripts/adv_graphgen/sanitize.sh28
-rwxr-xr-xretiolum/scripts/adv_graphgen/tinc_stats.py83
8 files changed, 312 insertions, 122 deletions
diff --git a/retiolum/bin/my-ip b/retiolum/bin/my-ip
new file mode 100755
index 00000000..fcfbba05
--- /dev/null
+++ b/retiolum/bin/my-ip
@@ -0,0 +1,2 @@
+#!/bin/sh
+curl http://euer.krebsco.de/live/ip.php
diff --git a/retiolum/bin/tinc_stats b/retiolum/bin/tinc_stats
new file mode 120000
index 00000000..6a58af60
--- /dev/null
+++ b/retiolum/bin/tinc_stats
@@ -0,0 +1 @@
+/home/makefu/repos/krebs/retiolum/scripts/adv_graphgen/tinc_stats.py \ No newline at end of file
diff --git a/retiolum/hosts/EUcancER b/retiolum/hosts/euer
index ae2d5f9d..ae2d5f9d 100644
--- a/retiolum/hosts/EUcancER
+++ b/retiolum/hosts/euer
diff --git a/retiolum/scripts/adv_graphgen/BackwardsReader.py b/retiolum/scripts/adv_graphgen/BackwardsReader.py
new file mode 100644
index 00000000..6bdbf43c
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/BackwardsReader.py
@@ -0,0 +1,35 @@
+import sys
+import os
+import string
+
+class BackwardsReader:
+ """ Stripped and stolen from : http://code.activestate.com/recipes/120686-read-a-text-file-backwards/ """
+ def readline(self):
+ while len(self.data) == 1 and ((self.blkcount * self.blksize) < self.size):
+ self.blkcount = self.blkcount + 1
+ line = self.data[0]
+ try:
+ self.f.seek(-self.blksize * self.blkcount, 2)
+ self.data = string.split(self.f.read(self.blksize) + line, '\n')
+ except IOError:
+ self.f.seek(0)
+ self.data = string.split(self.f.read(self.size - (self.blksize * (self.blkcount-1))) + line, '\n')
+
+ if len(self.data) == 0:
+ return ""
+
+ line = self.data[-1]
+ self.data = self.data[:-1]
+ return line + '\n'
+
+ def __init__(self, file, blksize=4096):
+ """initialize the internal structures"""
+ self.size = os.stat(file)[6]
+ self.blksize = blksize
+ self.blkcount = 1
+ self.f = open(file, 'rb')
+ if self.size > self.blksize:
+ self.f.seek(-self.blksize * self.blkcount, 2)
+ self.data = string.split(self.f.read(self.blksize), '\n')
+ if not self.data[-1]:
+ self.data = self.data[:-1]
diff --git a/retiolum/scripts/adv_graphgen/parse.py b/retiolum/scripts/adv_graphgen/parse.py
deleted file mode 100755
index aae69f0f..00000000
--- a/retiolum/scripts/adv_graphgen/parse.py
+++ /dev/null
@@ -1,110 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf8 -*-
-
-import sys
-""" TODO: Refactoring needed to pull the edges out of the node structures again,
-it should be easier to handle both structures"""
-
-def write_digraph(nodes):
- """
- writes the complete digraph in dot format
- """
- print ('digraph retiolum {')
- print (' node[shape=box,style=filled,fillcolor=grey]')
- print (' overlap=false')
- generate_stats(nodes)
- nodes = delete_unused_nodes(nodes)
- merge_edges(nodes)
- for k,v in nodes.iteritems():
- write_node(k,v)
- print ('}')
-def generate_stats(nodes):
- """ Generates some statistics of the network and nodes
- """
- for k,v in nodes.iteritems():
- v['num_conns'] = len(v.get('to',[]))
-def delete_unused_nodes(nodes):
- new_nodes = {}
- for k,v in nodes.iteritems():
- if v['external-ip'] == "(null)":
- continue
- if v.get('to',[]):
- new_nodes[k] = v
- for k,v in new_nodes.iteritems():
- if not [ i for i in v['to'] if i['name'] in new_nodes]:
- #del(new_nodes[k])
- del(k)
- return new_nodes
-def merge_edges(nodes):
- """ merge back and forth edges into one
- DESTRUCTS the current structure by deleting "connections" in the nodes
-
- """
- for k,v in nodes.iteritems():
- for con in v.get('to',[]):
- for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])):
- if k == secon['name']:
- del (nodes[con['name']]['to'][i])
- con['bidirectional'] = True
-
-
-def write_node(k,v):
- """ writes a single node and its edges
- edges are weightet with the informations inside the nodes provided by
- tinc
- """
-
- node = " "+k+"[label=\""
- node += k+"\\l"
- node += "external:"+v['external-ip']+":"+v['external-port']+"\\l"
- if v.has_key('num_conns'):
- node += "Num Connects:"+str(v['num_conns'])+"\\l"
- for addr in v.get('internal-ip',['¯\\\\(°_o)/¯']):
- node += "internal:"+addr+"\\l"
- node +="\""
- if v['external-ip'] == "MYSELF":
- node += ",fillcolor=steelblue1"
- node +=",group=\""+v['external-ip'].replace(".","")+"\""
- node += "]"
- print node
-
- for con in v.get('to',[]):
- edge = " "+k+ " -> " +con['name'] + "[weight="+str(float(con['weight']))
- if con.get('bidirectional',False):
- edge += ",dir=both"
- edge += "]"
- print edge
-
-def parse_input():
- nodes={}
- for line in sys.stdin:
- line = line.replace('\n','')
- if line == 'Nodes:':
- nodes={}
- for line in sys.stdin:
- if line == 'End of nodes.\n':
- break
- l = line.replace('\n','').split() #TODO unhack me
- nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] }
- if line == 'Subnet list:':
- for line in sys.stdin:
- if line == 'End of subnet list.\n':
- break
- l = line.replace('\n','').split()
- if not nodes[l[2]].get('internal-ip',False):
- nodes[l[2]]['internal-ip'] = []
- nodes[l[2]]['internal-ip'].append(l[0].split('#')[0])
- if line == 'Edges:':
- edges = {}
- for line in sys.stdin:
- if line == 'End of edges.\n':
- break
- l = line.replace('\n','').split()
-
- if not nodes[l[0]].has_key('to') :
- nodes[l[0]]['to'] = []
- nodes[l[0]]['to'].append(
- {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] })
- return nodes
-nodes = parse_input()
-write_digraph(nodes)
diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
new file mode 100755
index 00000000..410e5229
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
@@ -0,0 +1,175 @@
+#!/usr/bin/python
+# -*- coding: utf8 -*-
+
+import sys,json
+supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ]
+""" TODO: Refactoring needed to pull the edges out of the node structures again,
+it should be easier to handle both structures"""
+DUMP_FILE = "/krebs/db/availability"
+def write_digraph(nodes):
+ """
+ writes the complete digraph in dot format
+ """
+ print ('digraph retiolum {')
+ #print (' graph [center rankdir=LR packMode="clust"]')
+ print (' graph [center packMode="clust"]')
+ print (' node[shape=box,style=filled,fillcolor=grey]')
+ print (' overlap=false')
+ generate_stats(nodes)
+ merge_edges(nodes)
+ write_stat_node(nodes)
+ for k,v in nodes.iteritems():
+ write_node(k,v)
+ print ('}')
+def dump_graph(nodes):
+ from time import time
+ graph = {}
+ graph['nodes'] = nodes
+ graph['timestamp'] = time()
+ f = open(DUMP_FILE,'a')
+ json.dump(graph,f)
+ f.write('\n')
+ f.close()
+def write_stat_node(nodes):
+ ''' Write a `stats` node in the corner
+ This node contains infos about the current number of active nodes and connections inside the network
+ '''
+ num_conns = 0
+ num_nodes = len(nodes)
+ for k,v in nodes.iteritems():
+ num_conns+= len(v['to'])
+ node_text = " stats_node [label=\"Statistics\\l"
+ node_text += "Active Nodes: %s\\l" % num_nodes
+ node_text += "Connections : %s\\l" % num_conns
+ node_text += "\""
+ node_text += ",fillcolor=green"
+ node_text += "]"
+ print(node_text)
+
+def generate_stats(nodes):
+ """ Generates some statistics of the network and nodes
+ """
+ jlines = []
+ try:
+ f = open(DUMP_FILE,'r')
+ for line in f:
+ jlines.append(json.loads(line))
+ f.close()
+ except Exception,e:
+ pass
+ for k,v in nodes.iteritems():
+ conns = v.get('to',[])
+ v['num_conns'] = len(conns)
+ v['avg_weight'] = get_node_avg_weight(conns)
+ v['availability'] = get_node_availability(k,jlines)
+ sys.stderr.write( "%s -> %f\n" %(k ,v['availability']))
+def get_node_avg_weight(conns):
+ """ calculates the average weight for the given connections """
+ if not conns:
+ sys.syderr.write("get_node_avg_weight: connection parameter empty")
+ return 9001
+ else:
+ return sum([float(c['weight']) for c in conns])/len(conns)
+def get_node_availability(name,jlines):
+ """ calculates the node availability by reading the generated dump file
+ adding together the uptime of the node and returning the time
+ parms:
+ name - node name
+ jlines - list of already parsed dictionaries node archive
+ """
+ begin = last = current = 0
+ uptime = 0
+ #sys.stderr.write ( "Getting Node availability of %s\n" % name)
+ for stat in jlines:
+ if not stat['nodes']:
+ continue
+ ts = stat['timestamp']
+ if not begin:
+ begin = last = ts
+ current = ts
+ if stat['nodes'].get(name,{}).get('to',[]):
+ uptime += current - last
+ else:
+ pass
+ #sys.stderr.write("%s offline at timestamp %f\n" %(name,current))
+ last = ts
+ all_the_time = last - begin
+ try:
+ return uptime/ all_the_time
+ except:
+ return 1
+
+def delete_unused_nodes(nodes):
+ new_nodes = {}
+ for k,v in nodes.iteritems():
+ if v['external-ip'] == "(null)":
+ continue
+ if v.get('to',[]):
+ new_nodes[k] = v
+ for k,v in new_nodes.iteritems():
+ if not [ i for i in v['to'] if i['name'] in new_nodes]:
+ #del(new_nodes[k])
+ del(k)
+ return new_nodes
+def merge_edges(nodes):
+ """ merge back and forth edges into one
+ DESTRUCTS the current structure by deleting "connections" in the nodes
+ """
+ for k,v in nodes.iteritems():
+ for con in v.get('to',[]):
+ for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])):
+ if k == secon['name']:
+ del (nodes[con['name']]['to'][i])
+ con['bidirectional'] = True
+
+
+def write_node(k,v):
+ """ writes a single node and its edges
+ edges are weightet with the informations inside the nodes provided by
+ tinc
+ """
+
+ node = " "+k+"[label=\""
+ node += k+"\\l"
+ node += "availability: %f\\l" % v['availability']
+ node += "avg weight: %.2f\\l" % v['avg_weight']
+ if v.has_key('num_conns'):
+ node += "Num Connects:"+str(v['num_conns'])+"\\l"
+ node += "external:"+v['external-ip']+":"+v['external-port']+"\\l"
+ for addr in v.get('internal-ip',['¯\\\\(°_o)/¯']):
+ node += "internal:"+addr+"\\l"
+ node +="\""
+ if k in supernodes:
+ node += ",fillcolor=steelblue1"
+ #node +=",group=\""+v['external-ip'].replace(".","")+"\""
+ node += "]"
+ print node
+
+ for con in v.get('to',[]):
+ label = con['weight']
+ w = int(con['weight'])
+ weight = str(1000 - (((w - 150) * (1000 - 0)) / (1000 -150 )) + 0)
+
+ length = str(float(w)/1500)
+ #weight = "1000" #str(300/float(con['weight']))
+ #weight = str((100/float(con['weight'])))
+ #weight = str(-1 * (200-100000/int(con['weight'])))
+ if float(weight) < 0 :
+ weight= "1"
+
+ #sys.stderr.write(weight + ":"+ length +" %s -> " %k + str(con) + "\n")
+ edge = " "+k+ " -> " +con['name'] + "[label="+label + " weight="+weight #+ " minlen="+length
+ if con.get('bidirectional',False):
+ edge += ",dir=both"
+ edge += "]"
+ print edge
+
+def decode_input(FILE):
+ return json.load(FILE)
+nodes = decode_input(sys.stdin)
+nodes = delete_unused_nodes(nodes)
+try:
+ dump_graph(nodes)
+except Exception,e:
+ sys.stderr.write("Cannot dump graph: %s" % str(e))
+write_digraph(nodes)
diff --git a/retiolum/scripts/adv_graphgen/sanitize.sh b/retiolum/scripts/adv_graphgen/sanitize.sh
index 1dc43bf4..402ce256 100755
--- a/retiolum/scripts/adv_graphgen/sanitize.sh
+++ b/retiolum/scripts/adv_graphgen/sanitize.sh
@@ -1,20 +1,24 @@
#!/bin/sh
-HERE=$(dirname `readlink -f $0`)
-TMP=/tmp
+cd $(dirname `readlink -f $0`)
GRAPH_SETTER1=dot
GRAPH_SETTER2=circo
GRAPH_SETTER3='neato -Goverlap=prism '
GRAPH_SETTER4=sfdp
LOG_FILE=/var/log/syslog
+TYPE=svg
+TYPE2=png
OPENER=/bin/true
+DOTFILE=`mktemp`
+trap 'rm $DOTFILE' SIGTERM
+sudo LOG_FILE=$LOG_FILE python tinc_stats.py |\
+ python parse_tinc_stats.py > $DOTFILE
-sudo pkill -USR2 tincd
-sudo sed -n '/tinc.retiolum/{s/.*tinc.retiolum\[[0-9]*\]: //gp}' $LOG_FILE |\
- $HERE/parse.py > $TMP/retiolum.dot
-
-$GRAPH_SETTER1 -Tpng -o $1/retiolum_1.png $TMP/retiolum.dot
-$GRAPH_SETTER2 -Tpng -o $1/retiolum_2.png $TMP/retiolum.dot
-$GRAPH_SETTER3 -Tpng -o $1/retiolum_3.png $TMP/retiolum.dot
-$GRAPH_SETTER4 -Tpng -o $1/retiolum_4.png $TMP/retiolum.dot
-$OPENER $HERE/retiolum_1.png &>/dev/null
-rm $TMP/retiolum.dot
+$GRAPH_SETTER1 -T$TYPE -o $1/retiolum_1.$TYPE $DOTFILE
+$GRAPH_SETTER2 -T$TYPE -o $1/retiolum_2.$TYPE $DOTFILE
+$GRAPH_SETTER3 -T$TYPE -o $1/retiolum_3.$TYPE $DOTFILE
+$GRAPH_SETTER4 -T$TYPE -o $1/retiolum_4.$TYPE $DOTFILE
+#convert -resize 20% $1/retiolum_1.$TYPE $1/retiolum_1.$TYPE2
+#convert -resize 20% $1/retiolum_2.$TYPE $1/retiolum_2.$TYPE2
+#convert -resize 20% $1/retiolum_3.$TYPE $1/retiolum_3.$TYPE2
+#convert -resize 20% $1/retiolum_4.$TYPE $1/retiolum_4.$TYPE2
+#$OPENER $1/retiolum_1.$TYPE &>/dev/null
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats.py b/retiolum/scripts/adv_graphgen/tinc_stats.py
new file mode 100755
index 00000000..be3bbbff
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/tinc_stats.py
@@ -0,0 +1,83 @@
+#!/usr/bin/python
+from BackwardsReader import BackwardsReader
+import os
+import re
+import sys
+import json
+
+
+TINC_NETWORK = os.environ.get("TINC_NETWORK","retiolum")
+os.environ["LOG_FILE"]
+SYSLOG_FILE = os.environ.get("LOG_FILE","/var/log/everything.log")
+
+
+# Tags and Delimiters
+TINC_TAG="tinc.%s" % TINC_NETWORK
+BEGIN_NODES = "Nodes:"
+END_NODES = "End of nodes."
+BEGIN_SUBNET = "Subnet list:"
+END_SUBNET = "End of subnet list"
+BEGIN_EDGES = "Edges:"
+END_EDGES = "End of edges."
+
+def get_tinc_block(log_file):
+ """ returns an iterateable block from the given log file (syslog) """
+ tinc_block = []
+ in_block = False
+ bf = BackwardsReader(log_file)
+ BOL = re.compile(".*tinc.retiolum\[[0-9]+\]: ")
+ while True:
+ line = bf.readline()
+ if not line:
+ raise Exception("end of file at log file? This should not happen!")
+ line = BOL.sub('',line).strip()
+
+ if END_SUBNET in line:
+ in_block = True
+
+ if not in_block:
+ continue
+
+ tinc_block.append(line)
+
+ if BEGIN_NODES in line:
+ break
+ return reversed(tinc_block)
+
+def parse_input(log_data):
+ nodes={}
+ for line in log_data:
+ if BEGIN_NODES in line :
+ nodes={}
+ for line in log_data:
+ if END_NODES in line :
+ break
+ l = line.replace('\n','').split() #TODO unhack me
+ nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] }
+ if BEGIN_SUBNET in line :
+ for line in log_data:
+ if END_SUBNET in line :
+ break
+ l = line.replace('\n','').split()
+ if not nodes[l[2]].get('internal-ip',False):
+ nodes[l[2]]['internal-ip'] = []
+ nodes[l[2]]['internal-ip'].append(l[0].split('#')[0])
+ if BEGIN_EDGES in line :
+ edges = {}
+ for line in log_data:
+ if END_EDGES in line :
+ break
+ l = line.replace('\n','').split()
+
+ if not nodes[l[0]].has_key('to') :
+ nodes[l[0]]['to'] = []
+ nodes[l[0]]['to'].append(
+ {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] })
+ return nodes
+
+
+if __name__ == '__main__':
+ import subprocess,time
+ subprocess.popen("pkill -SIGUSR2 tincd")
+ time.sleep(1)
+ print json.dumps(parse_input((get_tinc_block(SYSLOG_FILE))))