summaryrefslogtreecommitdiffstats
path: root/retiolum/scripts
diff options
context:
space:
mode:
Diffstat (limited to 'retiolum/scripts')
-rw-r--r--retiolum/scripts/adv_graphgen/USAGE.md27
-rwxr-xr-xretiolum/scripts/adv_graphgen/all_the_graphs.sh4
-rwxr-xr-xretiolum/scripts/adv_graphgen/anonytize.sh1
-rwxr-xr-xretiolum/scripts/adv_graphgen/sanitize.sh1
-rwxr-xr-xretiolum/scripts/adv_graphgen/tinc_stats/Availability.py2
-rwxr-xr-x[-rw-r--r--]retiolum/scripts/adv_graphgen/tinc_stats/Graph.py48
-rw-r--r--retiolum/scripts/adv_graphgen/tinc_stats/Graphite.py24
-rwxr-xr-xretiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py59
-rwxr-xr-x[-rw-r--r--]retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py2
-rw-r--r--retiolum/scripts/adv_graphgen/tinc_stats/__init__.py1
10 files changed, 57 insertions, 112 deletions
diff --git a/retiolum/scripts/adv_graphgen/USAGE.md b/retiolum/scripts/adv_graphgen/USAGE.md
new file mode 100644
index 00000000..9fa541ed
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/USAGE.md
@@ -0,0 +1,27 @@
+# Example usage
+make sure you have the correct permissions!
+
+# High Level
+
+ # creates all graphs with predefined paths (see source code of this script)
+ ./all_the_graphs.sh
+
+ # create anonymized graphs to /tmp
+ ./anonytize.sh /tmp
+
+ # create full detail graphs to /var/www/graph.retiolum
+ ./sanitize.sh /var/ww/graph.retiolum
+
+ # return currently availabe supernodes
+ tinc_stats/Supernodes.py
+
+# Low Level
+
+ # returns the current tinc graph as json
+ tinc_stats/Log2JSON.py
+
+ # adds GEOIP information to the json file
+ tinc_stats/Log2JSON.py | tinc_stats/Geo.py
+
+ # creates a grapviz file from current graph, pipes into a timpfile
+ tinc_stats/Log2JSON.py | tinc_stats/Graph.py complete > /tmp/out.graphviz
diff --git a/retiolum/scripts/adv_graphgen/all_the_graphs.sh b/retiolum/scripts/adv_graphgen/all_the_graphs.sh
index 381c7aeb..9f3bf82b 100755
--- a/retiolum/scripts/adv_graphgen/all_the_graphs.sh
+++ b/retiolum/scripts/adv_graphgen/all_the_graphs.sh
@@ -4,17 +4,13 @@
echo "`date` begin all graphs" >> /tmp/build_graph
cd $(dirname $(readlink -f $0))
PATH=$PATH:../../../util/bin/
- export LOG_FILE=/var/log/tinc-retiolum.log
- export TINC_LEGACY=true
EXTERNAL_FOLDER=/var/www/euer.krebsco.de/graphs/retiolum
INTERNAL_FOLDER=/var/www/euer/graphs/retiolum
begin=`timer`
- export GRAPHITE_HOST="no-omo"
export GEOCTIYDB="$PWD/GeoLiteCity.dat"
(python tinc_stats/Log2JSON.py | python tinc_stats/Geo.py > $INTERNAL_FOLDER/marker.json)&
(./anonytize.sh $EXTERNAL_FOLDER && echo "`date` anonytize done" >> /tmp/build_graph)&
(./sanitize.sh $INTERNAL_FOLDER && echo "`date` sanitize done" >> /tmp/build_graph)&
# wait
- graphitec "retiolum.graph.buildtime" "$(timer $begin)" >> /tmp/build_graph
echo "`date` end all graphs" >> /tmp/build_graph
)&
diff --git a/retiolum/scripts/adv_graphgen/anonytize.sh b/retiolum/scripts/adv_graphgen/anonytize.sh
index 4c16daf9..04a68869 100755
--- a/retiolum/scripts/adv_graphgen/anonytize.sh
+++ b/retiolum/scripts/adv_graphgen/anonytize.sh
@@ -5,7 +5,6 @@ GRAPH_SETTER1=dot
GRAPH_SETTER2=circo
GRAPH_SETTER3='neato -Goverlap=prism '
GRAPH_SETTER4=sfdp
-LOG_FILE=${LOG_FILE:-/var/log/syslog}
TYPE=svg
TYPE2=png
OPENER=/bin/true
diff --git a/retiolum/scripts/adv_graphgen/sanitize.sh b/retiolum/scripts/adv_graphgen/sanitize.sh
index 8da6ec70..f7d0e7e7 100755
--- a/retiolum/scripts/adv_graphgen/sanitize.sh
+++ b/retiolum/scripts/adv_graphgen/sanitize.sh
@@ -5,7 +5,6 @@ GRAPH_SETTER1=dot
GRAPH_SETTER2=circo
GRAPH_SETTER3='neato -Goverlap=prism '
GRAPH_SETTER4=sfdp
-LOG_FILE=${LOG_FILE:-/var/log/syslog}
TYPE=svg
TYPE2=png
OPENER=/bin/true
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py b/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py
index a1ef13f1..66defa44 100755
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py
@@ -20,7 +20,7 @@ def generate_stats():
for line in f:
jlines.append(json.loads(line))
f.close()
- except Exception,e:
+ except Exception as e:
pass
all_nodes = {}
for k in get_all_nodes():
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py b/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py
index 18c3d545..9d80e62d 100644..100755
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py
@@ -1,6 +1,5 @@
#!/usr/bin/python
from BackwardsReader import BackwardsReader
-from Graphite import GraphiteSender
import sys,json,os
from Supernodes import check_all_the_super
from Availability import get_node_availability
@@ -11,9 +10,9 @@ DUMP_FILE = "/krebs/db/availability"
def resolve_myself(nodes):
#resolve MYSELF to the real ip
- for k,v in nodes.iteritems():
+ for k,v in nodes.items():
if v["external-ip"] == "MYSELF":
- for nodek,node in nodes.iteritems():
+ for nodek,node in nodes.items():
for to in node['to']:
if to['name'] == k:
v["external-ip"] = to["addr"]
@@ -45,16 +44,16 @@ def generate_availability_stats(nodes):
jlines.append(jline)
lines_to_use -=1
- except Exception,e: sys.stderr.write(str(e))
+ except Exception as e: sys.stderr.write(str(e))
- for k,v in nodes.iteritems():
+ for k,v in nodes.items():
v['availability'] = get_node_availability(k,jlines)
sys.stderr.write( "%s -> %f\n" %(k ,v['availability']))
def generate_stats(nodes):
""" Generates some statistics of the network and nodes
"""
- for k,v in nodes.iteritems():
+ for k,v in nodes.items():
conns = v.get('to',[])
for c in conns: #sanitize weights
if float(c['weight']) > 9000: c['weight'] = str(9001)
@@ -73,12 +72,12 @@ def get_node_avg_weight(conns):
def delete_unused_nodes(nodes):
""" Deletes all the nodes which are currently not connected to the network"""
new_nodes = {}
- for k,v in nodes.iteritems():
+ for k,v in nodes.items():
if v['external-ip'] == "(null)":
continue
if v.get('to',[]):
new_nodes[k] = v
- for k,v in new_nodes.iteritems():
+ for k,v in new_nodes.items():
if not [ i for i in v['to'] if i['name'] in new_nodes]:
del(k)
return new_nodes
@@ -87,7 +86,7 @@ def merge_edges(nodes):
""" merge back and forth edges into one
DESTRUCTS the current structure by deleting "connections" in the nodes
"""
- for k,v in nodes.iteritems():
+ for k,v in nodes.items():
for con in v.get('to',[]):
for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])):
if k == secon['name']:
@@ -97,7 +96,7 @@ def merge_edges(nodes):
def print_head():
print ('digraph retiolum {')
- print (' graph [center packMode="clust"]')
+ print (' graph [center=true packMode="clust"]')
print (' node[shape=box,style=filled,fillcolor=grey]')
print (' overlap=false')
@@ -112,8 +111,7 @@ def print_stat_node(nodes):
msg = '%s.num_nodes %d %d\r\n' %(g_path,num_nodes,begin)
s.send(msg)
except Exception as e: pass
- #except: pass
- for k,v in nodes.iteritems():
+ for k,v in nodes.items():
num_conns+= len(v['to'])
node_text = " stats_node [label=\"Statistics\\l"
node_text += "Build Date : %s\\l" % strftime("%Y-%m-%d %H:%M:%S",localtime())
@@ -133,7 +131,7 @@ def print_node(k,v):
node = " "+k+"[label=\""
node += k+"\\l"
node += "availability: %f\\l" % v['availability']
- if v.has_key('num_conns'):
+ if 'num_conns' in v:
node += "Num Connects:"+str(v['num_conns'])+"\\l"
node += "external:"+v['external-ip']+":"+v['external-port']+"\\l"
for addr in v.get('internal-ip',['dunno lol']):
@@ -145,7 +143,7 @@ def print_node(k,v):
elif k in supernodes:
node += ",fillcolor=steelblue1"
node += "]"
- print node
+ print(node)
def print_anonymous_node(k,v):
""" writes a single node and its edges
@@ -154,7 +152,7 @@ def print_anonymous_node(k,v):
"""
node = " "+k #+"[label=\""
- print node
+ print(node)
def print_edge(k,v):
for con in v.get('to',[]):
@@ -170,14 +168,14 @@ def print_edge(k,v):
if con.get('bidirectional',False):
edge += ",dir=both"
edge += "]"
- print edge
+ print(edge)
def anonymize_nodes(nodes):
#anonymizes all nodes
i = "0"
newnodes = {}
- for k,v in nodes.iteritems():
- for nodek,node in nodes.iteritems():
+ for k,v in nodes.items():
+ for nodek,node in nodes.items():
for to in node['to']:
if to['name'] == k:
to['name'] = i
@@ -187,11 +185,6 @@ def anonymize_nodes(nodes):
if __name__ == "__main__":
supernodes= []
- try:
- gr = GraphiteSender(os.environ.get("GRAPHITE_HOST","localhost"))
- begin = time()
- except Exception as e:
- sys.stderr.write( "Cannot connect to graphite: " + str(e))
if len(sys.argv) != 2 or sys.argv[1] not in ["anonymous","complete"]:
print("usage: %s (anonymous|complete)")
sys.exit(1)
@@ -206,7 +199,7 @@ if __name__ == "__main__":
if sys.argv[1] == "anonymous":
nodes = anonymize_nodes(nodes)
- for k,v in nodes.iteritems():
+ for k,v in nodes.items():
print_anonymous_node(k,v)
print_edge(k,v)
@@ -214,18 +207,15 @@ if __name__ == "__main__":
for supernode,addr in check_all_the_super():
supernodes.append(supernode)
generate_availability_stats(nodes)
- for k,v in nodes.iteritems():
+ for k,v in nodes.items():
print_node(k,v)
print_edge(k,v)
try:
dump_graph(nodes)
- except Exception,e:
+ except Exception as e:
sys.stderr.write("Cannot dump graph: %s" % str(e))
else:
pass
print_stat_node(nodes)
print ('}')
- try:
- gr.send("graph.anon_build_time",(time()-begin)*1000)
- except Exception as e: pass
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Graphite.py b/retiolum/scripts/adv_graphgen/tinc_stats/Graphite.py
deleted file mode 100644
index 5002d8e5..00000000
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Graphite.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/python
-
-import socket
-from time import time
-
-class GraphiteSender:
- def __init__(self,host,port=2003,prefix="retiolum"):
- self.host = host
- self.port = port
- self.prefix = prefix
- self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- self.sock.connect((host,port))
-
- def send(name,data):
- # construct a message for graphite, honor the configured prefix
- self.sock.send("%s.%s %d %d\r\n"%(self.prefix,name,data,time()))
-
- def send_raw(path,data):
- #ignore the configured prefix, just it to the path given
- self.sock.send("%s %d %d\r\n"%(path,data,time()))
-
-if __name__ == "__main__":
- import sys
- GraphiteSender(sys.argv[1]).send_raw(sys.argv[2],sys.argv[3])
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py b/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py
index 644cbc63..a81e2bef 100755
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py
@@ -41,11 +41,6 @@ def debug(func):
return with_debug
-def get_tinc_log_file():
- # TODO parse logfile from somewhere
- return os.environ.get("LOG_FILE","/var/log/everything.log")
-
-
def parse_tinc_stats():
import subprocess
from time import sleep
@@ -57,12 +52,6 @@ def parse_tinc_stats():
elif which("tincctl"):
return parse_new_input("tincctl")
#old tinc
- elif which("tincd"):
- # TODO refactor me
- subprocess.call(["pkill","-SIGUSR2", "tincd"])
- sleep(1)
- return parse_input(get_tinc_block(get_tinc_log_file()))
- #no tinc
else:
raise Exception("no tinc executable found!")
@@ -95,13 +84,16 @@ def get_tinc_block(log_file):
def parse_new_input(tinc_bin):
nodes = {}
- pnodes = subprocess.Popen([tinc_bin,"-n",TINC_NETWORK,"dump","reachable","nodes"], stdout=subprocess.PIPE).communicate()[0]
+ pnodes = subprocess.Popen(
+ [tinc_bin,"-n",TINC_NETWORK,"dump","reachable","nodes"],
+ stdout=subprocess.PIPE).communicate()[0].decode()
#pnodes = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","reachable","nodes"])
for line in pnodes.split('\n'):
if not line: continue
l = line.split()
nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] }
- psubnets = subprocess.check_output([tinc_bin,"-n",TINC_NETWORK,"dump","subnets"])
+ psubnets = subprocess.check_output(
+ [tinc_bin,"-n",TINC_NETWORK,"dump","subnets"]).decode()
for line in psubnets.split('\n'):
if not line: continue
l = line.split()
@@ -111,12 +103,13 @@ def parse_new_input(tinc_bin):
nodes[l[2]]['internal-ip'].append(l[0].split('#')[0])
except KeyError:
pass # node does not exist (presumably)
- pedges = subprocess.check_output([tinc_bin,"-n",TINC_NETWORK,"dump","edges"])
+ pedges = subprocess.check_output(
+ [tinc_bin,"-n",TINC_NETWORK,"dump","edges"]).decode()
for line in pedges.split('\n'):
if not line: continue
l = line.split()
try:
- if not nodes[l[0]].has_key('to') :
+ if not 'to' in nodes[l[0]] :
nodes[l[0]]['to'] = []
nodes[l[0]]['to'].append(
{'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] })
@@ -124,43 +117,9 @@ def parse_new_input(tinc_bin):
pass #node does not exist
return nodes
-#@debug
-def parse_input(log_data):
- nodes={}
- for line in log_data:
- if BEGIN_NODES in line :
- nodes={}
- for line in log_data:
- if END_NODES in line :
- break
- l = line.replace('\n','').split() #TODO unhack me
- nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] }
- if BEGIN_SUBNET in line :
- for line in log_data:
- if END_SUBNET in line :
- break
- l = line.replace('\n','').split()
- if not nodes[l[2]].get('internal-ip',False):
- nodes[l[2]]['internal-ip'] = []
- nodes[l[2]]['internal-ip'].append(l[0].split('#')[0])
- if BEGIN_EDGES in line :
- edges = {}
- for line in log_data:
- if END_EDGES in line :
- break
- l = line.replace('\n','').split()
- if not nodes[l[0]].has_key('to') :
- nodes[l[0]]['to'] = []
- nodes[l[0]]['to'].append(
- {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] })
- return nodes
-
-
if __name__ == '__main__':
- # TODO refactor me
from sys import argv
if len(argv) > 1:
usage()
else:
- print json.dumps(parse_tinc_stats())
-
+ print (json.dumps(parse_tinc_stats()))
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py b/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py
index ae0fae8f..7bb79708 100644..100755
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py
@@ -56,4 +56,4 @@ if __name__ == "__main__":
usage
"""
for host,addrs in check_all_the_super():
- print host,addrs
+ print(host,addrs)
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py b/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py
index d4a686bf..62f541d1 100644
--- a/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py
@@ -4,4 +4,3 @@ import Log2JSON
import Supernodes
import Geo
import Graph
-import Graphite
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-04 17:20:52 +0000