diff options
20 files changed, 320 insertions, 216 deletions
diff --git a/retiolum/scripts/adv_graphgen/.gitignore b/retiolum/scripts/adv_graphgen/.gitignore new file mode 100644 index 00000000..fbee25a5 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/.gitignore @@ -0,0 +1,5 @@ +bin/ +include/ +lib/ +*.egg-info/ +GeoLiteCity.dat diff --git a/retiolum/scripts/adv_graphgen/README b/retiolum/scripts/adv_graphgen/README deleted file mode 100644 index 082e0f2b..00000000 --- a/retiolum/scripts/adv_graphgen/README +++ /dev/null @@ -1,28 +0,0 @@ -The folder contains a number of scripts which provide a convenient way to -generate advanced graphs from the SIGUSR2 output of tinc. - -it currently contains the following files: - -sanitize.sh: - wrapper arond parse.py which filters the syslog file for all tinc - related lines and removes the status informations: - this means that - <code> - May 19 20:40:44 servarch dnsmasq[5382]: reading /etc/resolv.conf - May 19 20:41:38 servarch tinc.retiolum[4780]: Error looking up pa-sharepoint.informatik.ba-stuttgart.de port 655: Name or service not known - </code> - becomes - <code> - Error looking up pa-sharepoint.informatik.ba-stuttgart.de port 655: Name or service not known - </code> - and so on. - It also provides a wrapper around graphviz which automagically - generates graphs from the produced graph file - -parse.py: - reads from stdin the sanitized syslog file and prints a valid dot file - from the given output. - The parser module may also produce any other output (e.g. for dns - entries and so on) you will need to actually read and modify the source - in order to be able to do this. ~May the source be with you~ - diff --git a/retiolum/scripts/adv_graphgen/README.md b/retiolum/scripts/adv_graphgen/README.md new file mode 100644 index 00000000..0f3ee285 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/README.md @@ -0,0 +1,38 @@ +# Generate Graphs from tinc + +## Install +### Nix + + # tinc_pre is required: + nix-env -i -f tinc_graphs.nix + + ## e.g. in Retiolum: + ## krebs.retiolum.tinc = pkgs.tinc_pre +### Local + + python setup.py install + # also install graphviz,imagemagic for building graphs + + +### Usage: + +see source of the 2 builder scripts: + + #all-around-builder + # env: EXTERNAL_FOLDER, INTERNAL_FOLDER, GEODB, TINC_HOSTPATH + all-the-graphs + + # build actual graphs + build-graph + + # exported py scripts + tinc-stats2json # - parses tinc current state into json + tinc-build-graph # - transfers json to graph + copy-map # - copies map.html into $1 + add-geodata # - adds geodata to json + tinc-availability-stats # adds availability data to json + +## Geodb infos + +- http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz +- nix-env -iA geolite-legacy diff --git a/retiolum/scripts/adv_graphgen/all_the_graphs.sh b/retiolum/scripts/adv_graphgen/all_the_graphs.sh deleted file mode 100755 index 9f3bf82b..00000000 --- a/retiolum/scripts/adv_graphgen/all_the_graphs.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -( - echo "`date` begin all graphs" >> /tmp/build_graph - cd $(dirname $(readlink -f $0)) - PATH=$PATH:../../../util/bin/ - EXTERNAL_FOLDER=/var/www/euer.krebsco.de/graphs/retiolum - INTERNAL_FOLDER=/var/www/euer/graphs/retiolum - begin=`timer` - export GEOCTIYDB="$PWD/GeoLiteCity.dat" - (python tinc_stats/Log2JSON.py | python tinc_stats/Geo.py > $INTERNAL_FOLDER/marker.json)& - (./anonytize.sh $EXTERNAL_FOLDER && echo "`date` anonytize done" >> /tmp/build_graph)& - (./sanitize.sh $INTERNAL_FOLDER && echo "`date` sanitize done" >> /tmp/build_graph)& -# wait - echo "`date` end all graphs" >> /tmp/build_graph -)& diff --git a/retiolum/scripts/adv_graphgen/anonytize.sh b/retiolum/scripts/adv_graphgen/anonytize.sh deleted file mode 100755 index 04a68869..00000000 --- a/retiolum/scripts/adv_graphgen/anonytize.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -set -euf -cd $(dirname `readlink -f $0`) -GRAPH_SETTER1=dot -GRAPH_SETTER2=circo -GRAPH_SETTER3='neato -Goverlap=prism ' -GRAPH_SETTER4=sfdp -TYPE=svg -TYPE2=png -OPENER=/bin/true -DOTFILE=`mktemp --suffix=anon` -trap 'rm $DOTFILE' INT TERM KILL EXIT -sudo -E python tinc_stats/Log2JSON.py |\ - python tinc_stats/Graph.py anonymous> $DOTFILE - - -i=1 -for setter in dot circo 'neato -Goverlap=prism ' sfdp -do - tmpgraph=`mktemp --tmpdir=$1` - $setter -T$TYPE -o $tmpgraph $DOTFILE - chmod go+rx $tmpgraph - mv $tmpgraph $1/retiolum_$i.$TYPE - i=`expr $i + 1` -done -convert $1/retiolum_1.$TYPE $1/retiolum_1.$TYPE2 -#convert -resize 20% $1/retiolum_2.$TYPE $1/retiolum_2.$TYPE2 -#convert -resize 20% $1/retiolum_3.$TYPE $1/retiolum_3.$TYPE2 -#convert -resize 20% $1/retiolum_4.$TYPE $1/retiolum_4.$TYPE2 diff --git a/retiolum/scripts/adv_graphgen/sanitize.sh b/retiolum/scripts/adv_graphgen/sanitize.sh deleted file mode 100755 index f7d0e7e7..00000000 --- a/retiolum/scripts/adv_graphgen/sanitize.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh -set -euf -cd $(dirname `readlink -f $0`) -GRAPH_SETTER1=dot -GRAPH_SETTER2=circo -GRAPH_SETTER3='neato -Goverlap=prism ' -GRAPH_SETTER4=sfdp -TYPE=svg -TYPE2=png -OPENER=/bin/true -DOTFILE=`mktemp --suffix=san` -trap 'rm $DOTFILE' INT TERM KILL EXIT -sudo -E python tinc_stats/Log2JSON.py |\ - python tinc_stats/Graph.py complete > $DOTFILE - - -i=1 -for setter in dot circo 'neato -Goverlap=prism ' sfdp -do - tmpgraph=`mktemp --tmpdir=$1` - $setter -T$TYPE -o $tmpgraph $DOTFILE - chmod go+rx $tmpgraph - mv $tmpgraph $1/retiolum_$i.$TYPE - i=`expr $i + 1` -done - -convert $1/retiolum_1.$TYPE $1/retiolum_1.$TYPE2 -#convert -resize 20% $1/retiolum_2.$TYPE $1/retiolum_2.$TYPE2 -#convert -resize 20% $1/retiolum_3.$TYPE $1/retiolum_3.$TYPE2 -#convert -resize 20% $1/retiolum_4.$TYPE $1/retiolum_4.$TYPE2 -rm $DOTFILE diff --git a/retiolum/scripts/adv_graphgen/scripts/all-the-graphs b/retiolum/scripts/adv_graphgen/scripts/all-the-graphs new file mode 100755 index 00000000..889a9fe0 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/scripts/all-the-graphs @@ -0,0 +1,18 @@ +#!/bin/bash + +EXTERNAL_FOLDER=${EXTERNAL_FOLDER:-/var/www/euer.krebsco.de/graphs/retiolum} +INTERNAL_FOLDER=${INTERNAL_FOLDER:-/var/www/euer/graphs/retiolum} +export GEODB="${GEODB:-}" +export TINC_HOSTPATH=${TINC_HOSTPATH:-~/painload/retiolum/hosts} +mapfile="$INTERNAL_FOLDER/map.html" + +if test -n "$GEOCITYDB";then + if test ! -e "$mapfile";then + echo "copying map to $map.html" + copy-map "$mapfile" + fi + echo "creating geodata database" + tinc-stats2json | add-geodata > "$INTERNAL_FOLDER/marker.json" +fi +build-graphs anonymous "$EXTERNAL_FOLDER" +build-graphs complete "$INTERNAL_FOLDER" diff --git a/retiolum/scripts/adv_graphgen/scripts/build-graphs b/retiolum/scripts/adv_graphgen/scripts/build-graphs new file mode 100755 index 00000000..3eb675ec --- /dev/null +++ b/retiolum/scripts/adv_graphgen/scripts/build-graphs @@ -0,0 +1,27 @@ +#!/bin/sh +set -euf +mode=${1?arg1 must be either 'anonymous' or 'complete'} +out=${2?arg2 must be set to an outdir} +# for creating all the graphs +TYPE=svg +# for creating real pictures +TYPE2=png + +DOTFILE=`mktemp` +trap 'rm "$DOTFILE"' INT TERM KILL EXIT + +tinc-stats2json | tinc-build-graph "$mode" > "$DOTFILE" + +i=1 +for setter in dot circo 'neato -Goverlap=prism ' sfdp +do + tmpgraph=`mktemp --tmpdir=$1` + # first build, then move to avoid half-built graphs + "$setter" -T$TYPE -o "$tmpgraph" "$DOTFILE" ||: + chmod go+rx "$tmpgraph" + mv "$tmpgraph" "$1/retiolum_$i.$TYPE" + i=`expr $i + 1` +done + +convert -limit memory 32Mib -limit map 64Mib "$out/retiolum_1.$TYPE" "$out/retiolum_1.$TYPE2" +rm "$DOTFILE" diff --git a/retiolum/scripts/adv_graphgen/setup.py b/retiolum/scripts/adv_graphgen/setup.py new file mode 100644 index 00000000..dd772ef9 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/setup.py @@ -0,0 +1,42 @@ +import sys +from setuptools import setup + +setup( + name='tinc_graphs', + version='0.2.3', + + description='Create Graphs from tinc Stats', + long_description=open("README.md").read(), + license='WTFPL', + url='http://krebsco.de/', + download_url='https://pypi.python.org/pypi/tinc_graphs/', + + author='krebs', + author_email='spam@krebsco.de', + # you will also need graphviz and imagemagick + install_requires = [ 'pygeoip' ], + scripts = ['scripts/all-the-graphs', 'scripts/build-graphs'], + packages=['tinc_graphs'], + entry_points={ + 'console_scripts' : [ + 'tinc-stats2json = tinc_graphs.Log2JSON:main', + 'tinc-build-graph = tinc_graphs.Graph:main', + 'copy-map = tinc_graphs.Geo:copy_map', + 'add-geodata = tinc_graphs.Geo:main', + 'tinc-availability-stats = tinc_graphs.Availability:generate_stats', + ] + }, + + classifiers=[ + "Intended Audience :: Developers", + "Natural Language :: English", + "Operating System :: POSIX :: Linux", + "Development Status :: 3 - Alpha", + "Programming Language :: Python", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: Implementation :: CPython", + ], +) + diff --git a/retiolum/scripts/adv_graphgen/tinc_graphs.nix b/retiolum/scripts/adv_graphgen/tinc_graphs.nix new file mode 100644 index 00000000..662e81ac --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_graphs.nix @@ -0,0 +1,29 @@ +with import <nixpkgs> {}; +# nix-build Reaktor.nix +# result/bin/reaktor +## or in your env +# nix-env -i -f tinc_graphs.nix + +python3Packages.buildPythonPackage rec { + name = "tinc_graphs-${version}"; + version = "0.2.6"; + propagatedBuildInputs = with pkgs;[ + graphviz + imagemagick + + # optional if you want geolocation: + python3Packages.pygeoip + # geolite-legacy for the db: + ## ${geolite-legacy}/share/GeoIP/GeoIPCity.dat + ]; + #src = fetchurl { + #url = ""; + #sha256 = "1dksw1s1n2hxvnga6pygkr174dywncr0wiggkrkn1srbn2amh1c2"; + #}; + src = ./.; + meta = { + homepage = http://krebsco.de/; + description = "Create Graphs from Tinc Stats"; + license = stdenv.lib.licenses.wtfpl; + }; +} diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Availability.py index 66defa44..888335a7 100755 --- a/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Availability.py @@ -1,14 +1,15 @@ #!/usr/bin/python +# TODO: Rewrite this shitty piece of software ... # -*- coding: utf8 -*- -import sys,json +import sys,json,os """ TODO: Refactoring needed to pull the edges out of the node structures again, it should be easier to handle both structures""" -DUMP_FILE = "/krebs/db/availability" +DUMP_FILE = os.environ.get("AVAILABILITY_FILE","tinc-availability.json") +hostpath=os.environ.get("TINC_HOSTPATH", "/etc/tinc/retiolum/hosts") def get_all_nodes(): - import os - return os.listdir("/etc/tinc/retiolum/hosts") + return os.listdir(hostpath) def generate_stats(): """ Generates availability statistics of the network and nodes @@ -16,16 +17,18 @@ def generate_stats(): import json jlines = [] try: - f = open(DUMP_FILE,'r') + f = open(DUMP_FILE,'r+') for line in f: jlines.append(json.loads(line)) f.close() except Exception as e: - pass + print("Unable to open and parse Availability DB: {} (override with AVAILABILITY_FILE)".format(DUMP_FILE)) + sys.exit(1) + all_nodes = {} for k in get_all_nodes(): all_nodes[k] = get_node_availability(k,jlines) - print ( json.dumps(all_nodes)) + print (json.dumps(all_nodes)) def get_node_availability(name,jlines): """ calculates the node availability by reading the generated dump file diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py b/retiolum/scripts/adv_graphgen/tinc_graphs/BackwardsReader.py index 6bdbf43c..6bdbf43c 100644 --- a/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/BackwardsReader.py diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Geo.py index 038ca9c0..bfa4ee56 100755 --- a/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Geo.py @@ -1,26 +1,37 @@ #!/usr/bin/python3 # -*- coding: utf8 -*- import sys,json,os -from Graph import delete_unused_nodes,resolve_myself -GEODB=os.environ.get("GEOCITYDB","GeoLiteCity.dat") +from .Graph import delete_unused_nodes,resolve_myself +GEODB=os.environ.get("GEODB","GeoLiteCity.dat") + +def copy_map(): + from shutil import copy + from os.path import dirname,join,realpath + if len(sys.argv) != 2 or sys.argv[1] == "--help" : + print("usage: {} <destination>".format(sys.argv[0])) + print(" copies the map.html file to the <destination>") + sys.exit(1) + dstdir=sys.argv[1] + copy(realpath(join(dirname(__file__),'static/map.html')),dstdir) + def add_geo(nodes): from pygeoip import GeoIP gi = GeoIP(GEODB) - for k,v in nodes.iteritems(): + for k,v in nodes.items(): try: nodes[k].update(gi.record_by_addr(v["external-ip"])) except Exception as e: sys.stderr.write(str(e)) sys.stderr.write("Cannot determine GeoData for %s\n"%k) - return nodes + def add_coords_to_edges(nodes): from pygeoip import GeoIP gi = GeoIP(GEODB) - for k,v in nodes.iteritems(): + for k,v in nodes.items(): for i,j in enumerate(v.get("to",[])): data=gi.record_by_addr(j["addr"]) try: @@ -34,13 +45,13 @@ def add_jitter(nodes): from random import random #add a bit of jitter to all of the coordinates max_jitter=0.005 - for k,v in nodes.iteritems(): + for k,v in nodes.items(): jitter_lat= max_jitter -random()*max_jitter*2 jitter_long= max_jitter -random()*max_jitter*2 try: v["latitude"]= v["latitude"] + jitter_lat v["longitude"]= v["longitude"] + jitter_long - for nodek,node in nodes.iteritems(): + for nodek,node in nodes.items(): for to in node['to']: if to['name'] == k: to['latitude'] = v["latitude"] @@ -48,7 +59,20 @@ def add_jitter(nodes): except Exception as e: pass return nodes -if __name__ == "__main__": +def main(): import json - nodes = add_jitter(add_coords_to_edges(add_geo(resolve_myself(delete_unused_nodes(json.load(sys.stdin)))))) - print (json.dumps(nodes)) + try: + with open(GEODB,'rb') as f: f.read() + except Exception as e: + print("cannot open {} (GEODB in env)".format(GEODB)) + print(e) + sys.exit(1) + try: + nodes = add_jitter(add_coords_to_edges(add_geo(resolve_myself(delete_unused_nodes(json.load(sys.stdin)))))) + print (json.dumps(nodes)) + except Exception as e: + print("cannot parse data received via stdin") + print(e) + +if __name__ == "__main__": + main() diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Graph.py index 2fb09a58..29491997 100755 --- a/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Graph.py @@ -1,13 +1,16 @@ #!/usr/bin/python -from BackwardsReader import BackwardsReader +from .BackwardsReader import BackwardsReader import sys,json,os -from Supernodes import check_all_the_super -from Services import add_services -from Availability import get_node_availability +from .Supernodes import check_all_the_super +from .Services import add_services +from .Availability import get_node_availability import sys,json from time import time -DUMP_FILE = "/krebs/db/availability" +DUMP_FILE = os.environ.get("AVAILABILITY_FILE", "tinc-availability.json") +hostpath=os.environ.get("TINC_HOSTPATH", "/etc/tinc/retiolum/hosts") +# will be filled later +supernodes= [] def resolve_myself(nodes): #resolve MYSELF to the real ip @@ -19,6 +22,7 @@ def resolve_myself(nodes): v["external-ip"] = to["addr"] return nodes + def dump_graph(nodes): from time import time graph = {} @@ -33,23 +37,24 @@ def generate_availability_stats(nodes): """ generates stats of from availability """ jlines = [] - try: - f = BackwardsReader(DUMP_FILE) - lines_to_use = 1000 - while True: - if lines_to_use == 0: break - line = f.readline() - if not line: break - jline = json.loads(line) - if not jline['nodes']: continue - - jlines.append(jline) - lines_to_use -=1 - except Exception as e: sys.stderr.write(str(e)) + # try: + # f = BackwardsReader(DUMP_FILE) + # lines_to_use = 1000 + # while True: + # if lines_to_use == 0: break + # line = f.readline() + # if not line: break + # jline = json.loads(line) + # if not jline['nodes']: continue + + # jlines.append(jline) + # lines_to_use -=1 + # except Exception as e: sys.stderr.write(str(e)) for k,v in nodes.items(): - v['availability'] = get_node_availability(k,jlines) - sys.stderr.write( "%s -> %f\n" %(k ,v['availability'])) + # TODO: get this information in a different way + v['availability'] = get_node_availability(k,[]) + def generate_stats(nodes): """ Generates some statistics of the network and nodes @@ -209,8 +214,7 @@ def anonymize_nodes(nodes): i = str(int(i)+1) return newnodes -if __name__ == "__main__": - supernodes= [] +def main(): if len(sys.argv) != 2 or sys.argv[1] not in ["anonymous","complete"]: print("usage: %s (anonymous|complete)") sys.exit(1) @@ -230,21 +234,32 @@ if __name__ == "__main__": print_edge(k,v) elif sys.argv[1] == "complete": - for supernode,addr in check_all_the_super(): - supernodes.append(supernode) + try: + for supernode,addr in check_all_the_super(hostpath): + supernodes.append(supernode) + except FileNotFoundError as e: + print("!! cannot load list of supernodes ({})".format(hostpath)) + print("!! Use TINC_HOSTPATH env to override") + sys.exit(1) generate_availability_stats(nodes) add_services(nodes) for k,v in nodes.items(): print_node(k,v) print_edge(k,v) - try: - dump_graph(nodes) - except Exception as e: - sys.stderr.write("Cannot dump graph: %s" % str(e)) + + #TODO: get availability somehow else + # try: + # dump_graph(nodes) + # except Exception as e: + # sys.stderr.write("Cannot dump graph: %s" % str(e)) else: pass print_stat_node(nodes) print ('}') + +if __name__ == "__main__": + main() + # vim: set sw=2:ts=2 diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Log2JSON.py index a81e2bef..b0bc209b 100755 --- a/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Log2JSON.py @@ -26,12 +26,11 @@ def usage(): This tool dumps all tinc node informations as json ENVIRONMENT VARIABLES: - TINC_NETWORK The tinc network to dump + TINC_NETWORK The tinc network to dump (default: retiolum) - LOG_FILE If legacy tinc is used, defines the log file where tinc stats are dumped in - (default: /var/log/everything.log) """ % argv[0]) exit(1) + def debug(func): from functools import wraps @wraps(func) @@ -52,46 +51,35 @@ def parse_tinc_stats(): elif which("tincctl"): return parse_new_input("tincctl") #old tinc + elif which("tincd"): + print("old tincd not supported") + sys.exit(1) else: raise Exception("no tinc executable found!") - -#@debug -def get_tinc_block(log_file): - """ returns an iterateable block from the given log file (syslog) - This function became obsolete with the introduction of tincctl - """ - from BackwardsReader import BackwardsReader - tinc_block = [] - in_block = False - bf = BackwardsReader(log_file) - BOL = re.compile(".*tinc.%s\[[0-9]+\]: " % TINC_NETWORK) - while True: - line = bf.readline() - if not line: - raise Exception("end of file at log file? This should not happen!") - line = BOL.sub('',line).strip() - - if END_SUBNET in line: - in_block = True - - if not in_block: - continue - tinc_block.append(line) - - if BEGIN_NODES in line: - break - return reversed(tinc_block) + def parse_new_input(tinc_bin): - nodes = {} + nodes = {} pnodes = subprocess.Popen( [tinc_bin,"-n",TINC_NETWORK,"dump","reachable","nodes"], stdout=subprocess.PIPE).communicate()[0].decode() - #pnodes = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","reachable","nodes"]) for line in pnodes.split('\n'): if not line: continue l = line.split() - nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] } + n = l[0] + + token = l[1] + if token == 'id': + # new format + # <name> id <ident> at <ip> port <port> + ident = l[1] + l = l[2:] #shift over 'id <ident>' + # else: # token = 'at' + # old format: + # <name> at <ip> port <port> + _,_,ip,_,port = l[:5] + nodes[n]= { 'external-ip': ip, 'external-port' : l[4] } + psubnets = subprocess.check_output( [tinc_bin,"-n",TINC_NETWORK,"dump","subnets"]).decode() for line in psubnets.split('\n'): @@ -103,23 +91,29 @@ def parse_new_input(tinc_bin): nodes[l[2]]['internal-ip'].append(l[0].split('#')[0]) except KeyError: pass # node does not exist (presumably) + pedges = subprocess.check_output( [tinc_bin,"-n",TINC_NETWORK,"dump","edges"]).decode() for line in pedges.split('\n'): if not line: continue l = line.split() + # TODO: tokenize this and parse the line + n = l[0] try: if not 'to' in nodes[l[0]] : - nodes[l[0]]['to'] = [] - nodes[l[0]]['to'].append( - {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] }) + nodes[n]['to'] = [] + nodes[n]['to'].append( + {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[-1] }) except KeyError: pass #node does not exist return nodes -if __name__ == '__main__': +def main(): from sys import argv if len(argv) > 1: usage() else: print (json.dumps(parse_tinc_stats())) + +if __name__ == '__main__': + main() diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Services.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Services.py index 6752e116..9581e21e 100644 --- a/retiolum/scripts/adv_graphgen/tinc_stats/Services.py +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Services.py @@ -1,4 +1,5 @@ -services_dir="/home/makefu/r/services" +import os,sys +services_dir=os.environ.get("SERIVCES_DIR","/home/reaktor/nag.services") def add_services(nodes): for k,v in nodes.items(): n = nodes[k] @@ -10,8 +11,15 @@ def add_services(nodes): except Exception as e: n["services"] = ["Error: No Service File!"] return nodes -if __name__ == "__main__": +def main(): import json,sys - nodes = add_services(json.load(sys.stdin)) - print (json.dumps(nodes,indent=4)) + try: + nodes = add_services(json.load(sys.stdin)) + print (json.dumps(nodes,indent=4)) + except: + print("unable to parse json data from stdin") + sys.exit(1) + +if __name__ == "__main__": + main() # vim: set expandtab:ts=4:sw=4 diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Supernodes.py index 7e1f4dae..bc66b337 100755 --- a/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Supernodes.py @@ -7,21 +7,27 @@ def find_potential_super(path="/etc/tinc/retiolum/hosts"): needle_addr = re.compile("Address\s*=\s*(.*)") needle_port = re.compile("Port\s*=\s*(.*)") for f in os.listdir(path): - with open(path+"/"+f) as of: - addrs = [] - port = "655" + try: + with open(path+"/"+f) as of: + addrs = [] + port = "655" - for line in of.readlines(): + for line in of.readlines(): - addr_found = needle_addr.match(line) - if addr_found: - addrs.append(addr_found.group(1)) + addr_found = needle_addr.match(line) + if addr_found: + addrs.append(addr_found.group(1)) + + port_found = needle_port.match(line) + if port_found: + port = port_found.group(1) + + if addrs : yield (f ,[(addr ,int(port)) for addr in addrs]) + except FileNotFoundError as e: + print("Cannot open hosts directory to be used to find potential supernodes") + print("Directory used: {}".format(path)) + raise - port_found = needle_port.match(line) - if port_found: - port = port_found.group(1) - - if addrs : yield (f ,[(addr ,int(port)) for addr in addrs]) def try_connect(addr): try: @@ -34,7 +40,7 @@ def try_connect(addr): return addr except Exception as e: pass - #return () + def check_one_super(ha): host,addrs = ha @@ -44,16 +50,21 @@ def check_one_super(ha): if ret: valid_addrs.append(ret) if valid_addrs: return (host,valid_addrs) -def check_all_the_super(path="/etc/tinc/retiolum/hosts"): + +def check_all_the_super(path): from multiprocessing import Pool p = Pool(20) return filter(None,p.map(check_one_super,find_potential_super(path))) +def main(): + import os + hostpath=os.environ.get("TINC_HOSTPATH", "/etc/tinc/retiolum/hosts") -if __name__ == "__main__": - """ - usage - """ - for host,addrs in check_all_the_super(): + for host,addrs in check_all_the_super(hostpath): print("%s %s" %(host,str(addrs))) + +if __name__ == "__main__": + main() + +# vim: set expandtab:ts=:sw=2 diff --git a/retiolum/scripts/adv_graphgen/tinc_graphs/__init__.py b/retiolum/scripts/adv_graphgen/tinc_graphs/__init__.py new file mode 100644 index 00000000..414ffe99 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/__init__.py @@ -0,0 +1 @@ +__version__="0.2.3" diff --git a/retiolum/scripts/adv_graphgen/map.html b/retiolum/scripts/adv_graphgen/tinc_graphs/static/map.html index ef8a0565..ef8a0565 100644 --- a/retiolum/scripts/adv_graphgen/map.html +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/static/map.html diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py b/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py deleted file mode 100644 index f0fc8520..00000000 --- a/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -import Availability -import BackwardsReader -import Log2JSON -import Supernodes -import Geo -import Graph -import Services |