From 05bf2c8be209e619f8c9e727fd1353198b042642 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 9 Sep 2015 14:41:38 +0200 Subject: begin packaging of tinc_stats --- retiolum/scripts/adv_graphgen/.gitignore | 5 + retiolum/scripts/adv_graphgen/README | 28 --- retiolum/scripts/adv_graphgen/README.md | 28 +++ retiolum/scripts/adv_graphgen/all_the_graphs.sh | 16 -- retiolum/scripts/adv_graphgen/anonytize.sh | 29 --- retiolum/scripts/adv_graphgen/map.html | 88 ------- retiolum/scripts/adv_graphgen/sanitize.sh | 31 --- .../scripts/adv_graphgen/scripts/all-the-graphs | 13 ++ retiolum/scripts/adv_graphgen/scripts/build-graphs | 27 +++ retiolum/scripts/adv_graphgen/setup.py | 41 ++++ retiolum/scripts/adv_graphgen/tinc_graphs.nix | 25 ++ .../adv_graphgen/tinc_graphs/Availability.py | 60 +++++ .../adv_graphgen/tinc_graphs/BackwardsReader.py | 35 +++ retiolum/scripts/adv_graphgen/tinc_graphs/Geo.py | 77 +++++++ retiolum/scripts/adv_graphgen/tinc_graphs/Graph.py | 255 +++++++++++++++++++++ .../scripts/adv_graphgen/tinc_graphs/Log2JSON.py | 103 +++++++++ .../scripts/adv_graphgen/tinc_graphs/Services.py | 25 ++ .../scripts/adv_graphgen/tinc_graphs/Supernodes.py | 65 ++++++ .../scripts/adv_graphgen/tinc_graphs/__init__.py | 1 + .../adv_graphgen/tinc_graphs/static/map.html | 88 +++++++ .../adv_graphgen/tinc_stats/Availability.py | 58 ----- .../adv_graphgen/tinc_stats/BackwardsReader.py | 35 --- retiolum/scripts/adv_graphgen/tinc_stats/Geo.py | 54 ----- retiolum/scripts/adv_graphgen/tinc_stats/Graph.py | 250 -------------------- .../scripts/adv_graphgen/tinc_stats/Log2JSON.py | 125 ---------- .../scripts/adv_graphgen/tinc_stats/Services.py | 17 -- .../scripts/adv_graphgen/tinc_stats/Supernodes.py | 59 ----- .../scripts/adv_graphgen/tinc_stats/__init__.py | 7 - 28 files changed, 848 insertions(+), 797 deletions(-) create mode 100644 retiolum/scripts/adv_graphgen/.gitignore delete mode 100644 retiolum/scripts/adv_graphgen/README create mode 100644 retiolum/scripts/adv_graphgen/README.md delete mode 100755 retiolum/scripts/adv_graphgen/all_the_graphs.sh delete mode 100755 retiolum/scripts/adv_graphgen/anonytize.sh delete mode 100644 retiolum/scripts/adv_graphgen/map.html delete mode 100755 retiolum/scripts/adv_graphgen/sanitize.sh create mode 100755 retiolum/scripts/adv_graphgen/scripts/all-the-graphs create mode 100755 retiolum/scripts/adv_graphgen/scripts/build-graphs create mode 100644 retiolum/scripts/adv_graphgen/setup.py create mode 100644 retiolum/scripts/adv_graphgen/tinc_graphs.nix create mode 100755 retiolum/scripts/adv_graphgen/tinc_graphs/Availability.py create mode 100644 retiolum/scripts/adv_graphgen/tinc_graphs/BackwardsReader.py create mode 100755 retiolum/scripts/adv_graphgen/tinc_graphs/Geo.py create mode 100755 retiolum/scripts/adv_graphgen/tinc_graphs/Graph.py create mode 100755 retiolum/scripts/adv_graphgen/tinc_graphs/Log2JSON.py create mode 100644 retiolum/scripts/adv_graphgen/tinc_graphs/Services.py create mode 100755 retiolum/scripts/adv_graphgen/tinc_graphs/Supernodes.py create mode 100644 retiolum/scripts/adv_graphgen/tinc_graphs/__init__.py create mode 100644 retiolum/scripts/adv_graphgen/tinc_graphs/static/map.html delete mode 100755 retiolum/scripts/adv_graphgen/tinc_stats/Availability.py delete mode 100644 retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py delete mode 100755 retiolum/scripts/adv_graphgen/tinc_stats/Geo.py delete mode 100755 retiolum/scripts/adv_graphgen/tinc_stats/Graph.py delete mode 100755 retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py delete mode 100644 retiolum/scripts/adv_graphgen/tinc_stats/Services.py delete mode 100755 retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py delete mode 100644 retiolum/scripts/adv_graphgen/tinc_stats/__init__.py diff --git a/retiolum/scripts/adv_graphgen/.gitignore b/retiolum/scripts/adv_graphgen/.gitignore new file mode 100644 index 00000000..fbee25a5 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/.gitignore @@ -0,0 +1,5 @@ +bin/ +include/ +lib/ +*.egg-info/ +GeoLiteCity.dat diff --git a/retiolum/scripts/adv_graphgen/README b/retiolum/scripts/adv_graphgen/README deleted file mode 100644 index 082e0f2b..00000000 --- a/retiolum/scripts/adv_graphgen/README +++ /dev/null @@ -1,28 +0,0 @@ -The folder contains a number of scripts which provide a convenient way to -generate advanced graphs from the SIGUSR2 output of tinc. - -it currently contains the following files: - -sanitize.sh: - wrapper arond parse.py which filters the syslog file for all tinc - related lines and removes the status informations: - this means that - - May 19 20:40:44 servarch dnsmasq[5382]: reading /etc/resolv.conf - May 19 20:41:38 servarch tinc.retiolum[4780]: Error looking up pa-sharepoint.informatik.ba-stuttgart.de port 655: Name or service not known - - becomes - - Error looking up pa-sharepoint.informatik.ba-stuttgart.de port 655: Name or service not known - - and so on. - It also provides a wrapper around graphviz which automagically - generates graphs from the produced graph file - -parse.py: - reads from stdin the sanitized syslog file and prints a valid dot file - from the given output. - The parser module may also produce any other output (e.g. for dns - entries and so on) you will need to actually read and modify the source - in order to be able to do this. ~May the source be with you~ - diff --git a/retiolum/scripts/adv_graphgen/README.md b/retiolum/scripts/adv_graphgen/README.md new file mode 100644 index 00000000..082e0f2b --- /dev/null +++ b/retiolum/scripts/adv_graphgen/README.md @@ -0,0 +1,28 @@ +The folder contains a number of scripts which provide a convenient way to +generate advanced graphs from the SIGUSR2 output of tinc. + +it currently contains the following files: + +sanitize.sh: + wrapper arond parse.py which filters the syslog file for all tinc + related lines and removes the status informations: + this means that + + May 19 20:40:44 servarch dnsmasq[5382]: reading /etc/resolv.conf + May 19 20:41:38 servarch tinc.retiolum[4780]: Error looking up pa-sharepoint.informatik.ba-stuttgart.de port 655: Name or service not known + + becomes + + Error looking up pa-sharepoint.informatik.ba-stuttgart.de port 655: Name or service not known + + and so on. + It also provides a wrapper around graphviz which automagically + generates graphs from the produced graph file + +parse.py: + reads from stdin the sanitized syslog file and prints a valid dot file + from the given output. + The parser module may also produce any other output (e.g. for dns + entries and so on) you will need to actually read and modify the source + in order to be able to do this. ~May the source be with you~ + diff --git a/retiolum/scripts/adv_graphgen/all_the_graphs.sh b/retiolum/scripts/adv_graphgen/all_the_graphs.sh deleted file mode 100755 index 9f3bf82b..00000000 --- a/retiolum/scripts/adv_graphgen/all_the_graphs.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -( - echo "`date` begin all graphs" >> /tmp/build_graph - cd $(dirname $(readlink -f $0)) - PATH=$PATH:../../../util/bin/ - EXTERNAL_FOLDER=/var/www/euer.krebsco.de/graphs/retiolum - INTERNAL_FOLDER=/var/www/euer/graphs/retiolum - begin=`timer` - export GEOCTIYDB="$PWD/GeoLiteCity.dat" - (python tinc_stats/Log2JSON.py | python tinc_stats/Geo.py > $INTERNAL_FOLDER/marker.json)& - (./anonytize.sh $EXTERNAL_FOLDER && echo "`date` anonytize done" >> /tmp/build_graph)& - (./sanitize.sh $INTERNAL_FOLDER && echo "`date` sanitize done" >> /tmp/build_graph)& -# wait - echo "`date` end all graphs" >> /tmp/build_graph -)& diff --git a/retiolum/scripts/adv_graphgen/anonytize.sh b/retiolum/scripts/adv_graphgen/anonytize.sh deleted file mode 100755 index 04a68869..00000000 --- a/retiolum/scripts/adv_graphgen/anonytize.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -set -euf -cd $(dirname `readlink -f $0`) -GRAPH_SETTER1=dot -GRAPH_SETTER2=circo -GRAPH_SETTER3='neato -Goverlap=prism ' -GRAPH_SETTER4=sfdp -TYPE=svg -TYPE2=png -OPENER=/bin/true -DOTFILE=`mktemp --suffix=anon` -trap 'rm $DOTFILE' INT TERM KILL EXIT -sudo -E python tinc_stats/Log2JSON.py |\ - python tinc_stats/Graph.py anonymous> $DOTFILE - - -i=1 -for setter in dot circo 'neato -Goverlap=prism ' sfdp -do - tmpgraph=`mktemp --tmpdir=$1` - $setter -T$TYPE -o $tmpgraph $DOTFILE - chmod go+rx $tmpgraph - mv $tmpgraph $1/retiolum_$i.$TYPE - i=`expr $i + 1` -done -convert $1/retiolum_1.$TYPE $1/retiolum_1.$TYPE2 -#convert -resize 20% $1/retiolum_2.$TYPE $1/retiolum_2.$TYPE2 -#convert -resize 20% $1/retiolum_3.$TYPE $1/retiolum_3.$TYPE2 -#convert -resize 20% $1/retiolum_4.$TYPE $1/retiolum_4.$TYPE2 diff --git a/retiolum/scripts/adv_graphgen/map.html b/retiolum/scripts/adv_graphgen/map.html deleted file mode 100644 index ef8a0565..00000000 --- a/retiolum/scripts/adv_graphgen/map.html +++ /dev/null @@ -1,88 +0,0 @@ - - - - - - Simple Map - - - - - - - - - -
- - diff --git a/retiolum/scripts/adv_graphgen/sanitize.sh b/retiolum/scripts/adv_graphgen/sanitize.sh deleted file mode 100755 index f7d0e7e7..00000000 --- a/retiolum/scripts/adv_graphgen/sanitize.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh -set -euf -cd $(dirname `readlink -f $0`) -GRAPH_SETTER1=dot -GRAPH_SETTER2=circo -GRAPH_SETTER3='neato -Goverlap=prism ' -GRAPH_SETTER4=sfdp -TYPE=svg -TYPE2=png -OPENER=/bin/true -DOTFILE=`mktemp --suffix=san` -trap 'rm $DOTFILE' INT TERM KILL EXIT -sudo -E python tinc_stats/Log2JSON.py |\ - python tinc_stats/Graph.py complete > $DOTFILE - - -i=1 -for setter in dot circo 'neato -Goverlap=prism ' sfdp -do - tmpgraph=`mktemp --tmpdir=$1` - $setter -T$TYPE -o $tmpgraph $DOTFILE - chmod go+rx $tmpgraph - mv $tmpgraph $1/retiolum_$i.$TYPE - i=`expr $i + 1` -done - -convert $1/retiolum_1.$TYPE $1/retiolum_1.$TYPE2 -#convert -resize 20% $1/retiolum_2.$TYPE $1/retiolum_2.$TYPE2 -#convert -resize 20% $1/retiolum_3.$TYPE $1/retiolum_3.$TYPE2 -#convert -resize 20% $1/retiolum_4.$TYPE $1/retiolum_4.$TYPE2 -rm $DOTFILE diff --git a/retiolum/scripts/adv_graphgen/scripts/all-the-graphs b/retiolum/scripts/adv_graphgen/scripts/all-the-graphs new file mode 100755 index 00000000..3368d145 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/scripts/all-the-graphs @@ -0,0 +1,13 @@ +#!/bin/bash + +EXTERNAL_FOLDER=${EXTERNAL_FOLDER:-/var/www/euer.krebsco.de/graphs/retiolum} +INTERNAL_FOLDER=${INTERNAL_FOLDER:-/var/www/euer/graphs/retiolum} +GEOCTIYDB="${GEOCITYDB:-}" + +if test -n "$GEOCITYDB";then + # creates a marker file + # TODO: copy map.html from + tinc-stats2json | add-geodata > "$INTERNAL_FOLDER/marker.json" +fi +build-graphs anonymous "$EXTERNAL_FOLDER" +build-graphs complete "$INTERNAL_FOLDER" diff --git a/retiolum/scripts/adv_graphgen/scripts/build-graphs b/retiolum/scripts/adv_graphgen/scripts/build-graphs new file mode 100755 index 00000000..3eb675ec --- /dev/null +++ b/retiolum/scripts/adv_graphgen/scripts/build-graphs @@ -0,0 +1,27 @@ +#!/bin/sh +set -euf +mode=${1?arg1 must be either 'anonymous' or 'complete'} +out=${2?arg2 must be set to an outdir} +# for creating all the graphs +TYPE=svg +# for creating real pictures +TYPE2=png + +DOTFILE=`mktemp` +trap 'rm "$DOTFILE"' INT TERM KILL EXIT + +tinc-stats2json | tinc-build-graph "$mode" > "$DOTFILE" + +i=1 +for setter in dot circo 'neato -Goverlap=prism ' sfdp +do + tmpgraph=`mktemp --tmpdir=$1` + # first build, then move to avoid half-built graphs + "$setter" -T$TYPE -o "$tmpgraph" "$DOTFILE" ||: + chmod go+rx "$tmpgraph" + mv "$tmpgraph" "$1/retiolum_$i.$TYPE" + i=`expr $i + 1` +done + +convert -limit memory 32Mib -limit map 64Mib "$out/retiolum_1.$TYPE" "$out/retiolum_1.$TYPE2" +rm "$DOTFILE" diff --git a/retiolum/scripts/adv_graphgen/setup.py b/retiolum/scripts/adv_graphgen/setup.py new file mode 100644 index 00000000..84597def --- /dev/null +++ b/retiolum/scripts/adv_graphgen/setup.py @@ -0,0 +1,41 @@ +import sys +from setuptools import setup + +setup( + name='tinc_graphs', + version='0.2.3', + + description='Create Graphs from tinc Stats', + long_description=open("README.md").read(), + license='WTFPL', + url='http://krebsco.de/', + download_url='https://pypi.python.org/pypi/tinc_graphs/', + + author='krebs', + author_email='spam@krebsco.de', + # you will also need graphviz and imagemagick + install_requires = [ 'pygeoip' ], + + packages=['tinc_graphs'], + entry_points={ + 'console_scripts' : [ + 'tinc-stats2json = tinc_graphs.Log2JSON:main', + 'tinc-build-graph = tinc_graphs.Graph:main', + 'add-geodata = tinc_graphs.Geo:main', + 'tinc-availability-stats = tinc_graphs.Availability:generate_stats', + ] + }, + + classifiers=[ + "Intended Audience :: Developers", + "Natural Language :: English", + "Operating System :: POSIX :: Linux", + "Development Status :: 3 - Alpha", + "Programming Language :: Python", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: Implementation :: CPython", + ], +) + diff --git a/retiolum/scripts/adv_graphgen/tinc_graphs.nix b/retiolum/scripts/adv_graphgen/tinc_graphs.nix new file mode 100644 index 00000000..b45fde13 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_graphs.nix @@ -0,0 +1,25 @@ +with import {}; +# nix-build Reaktor.nix +# result/bin/reaktor +## or in your env +# nix-env -i -f tinc_graphs.nix + +buildPythonPackage rec { + name = "tinc_graphs-${version}"; + version = "0.2.6"; + propagatedBuildInputs = with pkgs;[ + pythonPackages.docopt + graphviz + imagemagick + pythonPackages.pygeoip + ]; + src = fetchurl { + url = ""; + sha256 = "1dksw1s1n2hxvnga6pygkr174dywncr0wiggkrkn1srbn2amh1c2"; + }; + meta = { + homepage = http://krebsco.de/; + description = "Create Graphs from Tinc Stats"; + license = stdenv.lib.licenses.wtfpl; + }; +} diff --git a/retiolum/scripts/adv_graphgen/tinc_graphs/Availability.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Availability.py new file mode 100755 index 00000000..42f28ef9 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Availability.py @@ -0,0 +1,60 @@ +#!/usr/bin/python +# -*- coding: utf8 -*- + +import sys,json,os +""" TODO: Refactoring needed to pull the edges out of the node structures again, +it should be easier to handle both structures""" +DUMP_FILE = os.environment.get("AVAILABILITY_FILE","tinc-availability.json") +hostpath=os.environment.get("TINC_HOSTPATH", "/etc/tinc/retiolum/hosts") + +def get_all_nodes(): + return os.listdir(hostpath) + +def generate_stats(): + """ Generates availability statistics of the network and nodes + """ + import json + jlines = [] + try: + f = open(DUMP_FILE,'r') + for line in f: + jlines.append(json.loads(line)) + f.close() + except Exception as e: + print("Unable to open and parse Availability DB: {} (override with AVAILABILITY_FILE)".format(DUMP_FILE) + sys.exit(1) + + all_nodes = {} + for k in get_all_nodes(): + all_nodes[k] = get_node_availability(k,jlines) + print (json.dumps(all_nodes)) + +def get_node_availability(name,jlines): + """ calculates the node availability by reading the generated dump file + adding together the uptime of the node and returning the time + parms: + name - node name + jlines - list of already parsed dictionaries node archive + """ + begin = last = current = 0 + uptime = 0 + for stat in jlines: + if not stat['nodes']: + continue + ts = stat['timestamp'] + if not begin: + begin = last = ts + current = ts + if stat['nodes'].get(name,{}).get('to',[]): + uptime += current - last + else: + pass + last = ts + all_the_time = last - begin + try: + return uptime/ all_the_time + except: + return 1 + +if __name__ == "__main__": + generate_stats() diff --git a/retiolum/scripts/adv_graphgen/tinc_graphs/BackwardsReader.py b/retiolum/scripts/adv_graphgen/tinc_graphs/BackwardsReader.py new file mode 100644 index 00000000..6bdbf43c --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/BackwardsReader.py @@ -0,0 +1,35 @@ +import sys +import os +import string + +class BackwardsReader: + """ Stripped and stolen from : http://code.activestate.com/recipes/120686-read-a-text-file-backwards/ """ + def readline(self): + while len(self.data) == 1 and ((self.blkcount * self.blksize) < self.size): + self.blkcount = self.blkcount + 1 + line = self.data[0] + try: + self.f.seek(-self.blksize * self.blkcount, 2) + self.data = string.split(self.f.read(self.blksize) + line, '\n') + except IOError: + self.f.seek(0) + self.data = string.split(self.f.read(self.size - (self.blksize * (self.blkcount-1))) + line, '\n') + + if len(self.data) == 0: + return "" + + line = self.data[-1] + self.data = self.data[:-1] + return line + '\n' + + def __init__(self, file, blksize=4096): + """initialize the internal structures""" + self.size = os.stat(file)[6] + self.blksize = blksize + self.blkcount = 1 + self.f = open(file, 'rb') + if self.size > self.blksize: + self.f.seek(-self.blksize * self.blkcount, 2) + self.data = string.split(self.f.read(self.blksize), '\n') + if not self.data[-1]: + self.data = self.data[:-1] diff --git a/retiolum/scripts/adv_graphgen/tinc_graphs/Geo.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Geo.py new file mode 100755 index 00000000..78add918 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Geo.py @@ -0,0 +1,77 @@ +#!/usr/bin/python3 +# -*- coding: utf8 -*- +import sys,json,os +from .Graph import delete_unused_nodes,resolve_myself +GEODB=os.environ.get("GEOCITYDB","GeoLiteCity.dat") + +def copy_map(): + from shutil import copytree + from os.path import dirname,join,realpath + if len(sys.argv) != 2 or sys.argv[1] == "--help" : + print("usage: {} ".format(sys.argv[0]) + print(" copies the map.html file to the ") + sys.exit(1) + dstdir=sys.argv[1] + copytree(realpath(join(dirname(__file__),'static/map.html')),dstdir) + + +def add_geo(nodes): + from pygeoip import GeoIP + gi = GeoIP(GEODB) + + for k,v in nodes.iteritems(): + try: + nodes[k].update(gi.record_by_addr(v["external-ip"])) + except Exception as e: + sys.stderr.write(str(e)) + sys.stderr.write("Cannot determine GeoData for %s\n"%k) + return nodes + +def add_coords_to_edges(nodes): + from pygeoip import GeoIP + gi = GeoIP(GEODB) + + for k,v in nodes.iteritems(): + for i,j in enumerate(v.get("to",[])): + data=gi.record_by_addr(j["addr"]) + try: + j["latitude"]=data["latitude"] + j["longitude"]=data["longitude"] + except Exception as e: pass + + return nodes + +def add_jitter(nodes): + from random import random + #add a bit of jitter to all of the coordinates + max_jitter=0.005 + for k,v in nodes.iteritems(): + jitter_lat= max_jitter -random()*max_jitter*2 + jitter_long= max_jitter -random()*max_jitter*2 + try: + v["latitude"]= v["latitude"] + jitter_lat + v["longitude"]= v["longitude"] + jitter_long + for nodek,node in nodes.iteritems(): + for to in node['to']: + if to['name'] == k: + to['latitude'] = v["latitude"] + to['longitude'] = v["longitude"] + except Exception as e: pass + return nodes + +def main(): + import json + try: + with open(GEODB) as f: f.read() + except: + print("cannot open {} (GEODB in env)".format(GEODB)) + sys.exit(1) + try: + nodes = add_jitter(add_coords_to_edges(add_geo(resolve_myself(delete_unused_nodes(json.load(sys.stdin)))))) + print (json.dumps(nodes)) + except Exception as e: + print("cannot parse data received via stdin") + print(e) + +if __name__ == "__main__": + main() diff --git a/retiolum/scripts/adv_graphgen/tinc_graphs/Graph.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Graph.py new file mode 100755 index 00000000..b001780d --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Graph.py @@ -0,0 +1,255 @@ +#!/usr/bin/python +from .BackwardsReader import BackwardsReader +import sys,json,os +from .Supernodes import check_all_the_super +from .Services import add_services +from .Availability import get_node_availability +import sys,json +from time import time +DUMP_FILE = os.environment.get("AVAILABILITY_FILE", "tinc-availability.json") + + +def resolve_myself(nodes): + #resolve MYSELF to the real ip + for k,v in nodes.items(): + if v["external-ip"] == "MYSELF": + for nodek,node in nodes.items(): + for to in node['to']: + if to['name'] == k: + v["external-ip"] = to["addr"] + return nodes + + +def dump_graph(nodes): + from time import time + graph = {} + graph['nodes'] = nodes + graph['timestamp'] = time() + f = open(DUMP_FILE,'a') + json.dump(graph,f) + f.write('\n') + f.close() + +def generate_availability_stats(nodes): + """ generates stats of from availability + """ + jlines = [] + try: + f = BackwardsReader(DUMP_FILE) + lines_to_use = 1000 + while True: + if lines_to_use == 0: break + line = f.readline() + if not line: break + jline = json.loads(line) + if not jline['nodes']: continue + + jlines.append(jline) + lines_to_use -=1 + except Exception as e: sys.stderr.write(str(e)) + + for k,v in nodes.items(): + v['availability'] = get_node_availability(k,jlines) + sys.stderr.write( "%s -> %f\n" %(k ,v['availability'])) + +def generate_stats(nodes): + """ Generates some statistics of the network and nodes + """ + for k,v in nodes.items(): + conns = v.get('to',[]) + for c in conns: #sanitize weights + if float(c['weight']) > 9000: c['weight'] = str(9001) + elif float(c['weight']) < 0: c['weight'] = str(0) + v['num_conns'] = len(conns) + v['avg_weight'] = get_node_avg_weight(conns) + +def get_node_avg_weight(conns): + """ calculates the average weight for the given connections """ + if not conns: + sys.syderr.write("get_node_avg_weight: connection parameter empty") + return 9001 + else: + return sum([float(c['weight']) for c in conns])/len(conns) + +def delete_unused_nodes(nodes): + """ Deletes all the nodes which are currently not connected to the network""" + new_nodes = {} + for k,v in nodes.items(): + if v['external-ip'] == "(null)": + continue + if v.get('to',[]): + new_nodes[k] = v + for k,v in new_nodes.items(): + if not [ i for i in v['to'] if i['name'] in new_nodes]: + del(k) + return new_nodes + +def merge_edges(nodes): + """ merge back and forth edges into one + DESTRUCTS the current structure by deleting "connections" in the nodes + """ + for k,v in nodes.items(): + for con in v.get('to',[]): + for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])): + if k == secon['name']: + del (nodes[con['name']]['to'][i]) + con['bidirectional'] = True + + +def print_head(): + print ('digraph retiolum {') + print (' graph [center=true packMode="clust"]') + print (' node[shape=box,style=filled,fillcolor=grey]') + print (' overlap=false') + +def print_stat_node(nodes): + ''' Write a `stats` node in the corner + This node contains infos about the current number of active nodes and connections inside the network + ''' + from time import localtime,strftime + num_conns = 0 + num_nodes = len(nodes) + try: + msg = '%s.num_nodes %d %d\r\n' %(g_path,num_nodes,begin) + s.send(msg) + except Exception as e: pass + for k,v in nodes.items(): + num_conns+= len(v['to']) + node_text = " stats_node [label=\"Statistics\\l" + node_text += "Build Date : %s\\l" % strftime("%Y-%m-%d %H:%M:%S",localtime()) + node_text += "Active Nodes: %s\\l" % num_nodes + node_text += "Connections : %s\\l" % num_conns + node_text += "\"" + node_text += ",fillcolor=green" + node_text += "]" + print(node_text) + +def print_node(k,v): + """ writes a single node and its edges + edges are weightet with the informations inside the nodes provided by + tinc + """ + + node = " "+k+"[label=<" %k + node += ""%k + if 'availability' in v: + node += "" % v['availability'] + + if 'num_conns' in v: + node += ""%str(v['num_conns']) + + node += "" + for addr in v.get('internal-ip',['dunno lol']): + node += ""%addr + + if 'services' in v: + node +="" + for service in v['services']: + try:uri,comment = service.split(" ",1) + except: + uri = service + comment ="" + node +="" + uri_proto=uri.split(":")[0] + uri_rest = uri.split(":")[1] + if not uri_rest: + node +="" + # end label + node +="
%s
availability:%f
Num Connects:%s
external:"+v['external-ip']+":"+v['external-port']+"
internal:%s
Services:
{0}".format(uri) + else: + node +="{0}".format(uri) + if comment: + node += "
{0}".format(comment) + node +="
>" + + if v['num_conns'] == 1: + node += ",fillcolor=red" + elif k in supernodes: + node += ",fillcolor=steelblue1" + node += "]" + + print(node) + +def print_anonymous_node(k,v): + """ writes a single node and its edges + edges are weightet with the informations inside the nodes provided by + tinc + """ + + node = " "+k #+"[label=\"" + print(node) + +def print_edge(k,v): + for con in v.get('to',[]): + label = con['weight'] + w = int(con['weight']) + weight = str(1000 - (((w - 150) * (1000 - 0)) / (1000 -150 )) + 0) + + length = str(float(w)/1500) + if float(weight) < 0 : + weight= "1" + + edge = " "+k+ " -> " +con['name'] + " [label="+label + " weight="+weight + if con.get('bidirectional',False): + edge += ",dir=both" + edge += "]" + print(edge) + +def anonymize_nodes(nodes): + #anonymizes all nodes + i = "0" + newnodes = {} + for k,v in nodes.items(): + for nodek,node in nodes.items(): + for to in node['to']: + if to['name'] == k: + to['name'] = i + newnodes[i] = v + i = str(int(i)+1) + return newnodes + +def main(): + supernodes= [] + if len(sys.argv) != 2 or sys.argv[1] not in ["anonymous","complete"]: + print("usage: %s (anonymous|complete)") + sys.exit(1) + + nodes = json.load(sys.stdin) + nodes = delete_unused_nodes(nodes) + print_head() + generate_stats(nodes) + merge_edges(nodes) + + + if sys.argv[1] == "anonymous": + nodes = anonymize_nodes(nodes) + + for k,v in nodes.items(): + print_anonymous_node(k,v) + print_edge(k,v) + + elif sys.argv[1] == "complete": + for supernode,addr in check_all_the_super(): + supernodes.append(supernode) + + generate_availability_stats(nodes) + add_services(nodes) + for k,v in nodes.items(): + print_node(k,v) + print_edge(k,v) + try: + dump_graph(nodes) + except Exception as e: + sys.stderr.write("Cannot dump graph: %s" % str(e)) + else: + pass + + print_stat_node(nodes) + print ('}') + +if __name__ == "__main__": + main() + +# vim: set sw=2:ts=2 diff --git a/retiolum/scripts/adv_graphgen/tinc_graphs/Log2JSON.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Log2JSON.py new file mode 100755 index 00000000..7df042fd --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Log2JSON.py @@ -0,0 +1,103 @@ +#!/usr/bin/python +import subprocess +import os +import re +import sys +import json + + +TINC_NETWORK =os.environ.get("TINC_NETWORK","retiolum") + +# is_legacy is the parameter which defines if the tinc config files are handled old fashioned (parse from syslog), +# or if the new and hip tincctl should be used + + +# Tags and Delimiters +TINC_TAG="tinc.%s" % TINC_NETWORK +BEGIN_NODES = "Nodes:" +END_NODES = "End of nodes." +BEGIN_SUBNET = "Subnet list:" +END_SUBNET = "End of subnet list" +BEGIN_EDGES = "Edges:" +END_EDGES = "End of edges." +def usage(): + from sys import argv,exit + print("""usage: %s +This tool dumps all tinc node informations as json + +ENVIRONMENT VARIABLES: + TINC_NETWORK The tinc network to dump + (default: retiolum) +""" % argv[0]) + exit(1) + +def debug(func): + from functools import wraps + @wraps(func) + def with_debug(*args,**kwargs): + print( func.__name__ + " (args: %s | kwargs %s)"% (args,kwargs)) + return func(*args,**kwargs) + return with_debug + + +def parse_tinc_stats(): + import subprocess + from time import sleep + from distutils.spawn import find_executable as which + #newest tinc + if which("tinc"): + return parse_new_input("tinc") + #new tinc + elif which("tincctl"): + return parse_new_input("tincctl") + #old tinc + elif which("tincd"): + print("old tincd not supported") + sys.exit(1) + else: + raise Exception("no tinc executable found!") + + +def parse_new_input(tinc_bin): + nodes = {} + pnodes = subprocess.Popen( + [tinc_bin,"-n",TINC_NETWORK,"dump","reachable","nodes"], + stdout=subprocess.PIPE).communicate()[0].decode() + for line in pnodes.split('\n'): + if not line: continue + l = line.split() + nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] } + psubnets = subprocess.check_output( + [tinc_bin,"-n",TINC_NETWORK,"dump","subnets"]).decode() + for line in psubnets.split('\n'): + if not line: continue + l = line.split() + try: + if not nodes[l[2]].get('internal-ip',False): + nodes[l[2]]['internal-ip'] = [] + nodes[l[2]]['internal-ip'].append(l[0].split('#')[0]) + except KeyError: + pass # node does not exist (presumably) + pedges = subprocess.check_output( + [tinc_bin,"-n",TINC_NETWORK,"dump","edges"]).decode() + for line in pedges.split('\n'): + if not line: continue + l = line.split() + try: + if not 'to' in nodes[l[0]] : + nodes[l[0]]['to'] = [] + nodes[l[0]]['to'].append( + {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] }) + except KeyError: + pass #node does not exist + return nodes + +def main(): + from sys import argv + if len(argv) > 1: + usage() + else: + print (json.dumps(parse_tinc_stats())) + +if __name__ == '__main__': + main() diff --git a/retiolum/scripts/adv_graphgen/tinc_graphs/Services.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Services.py new file mode 100644 index 00000000..227e5aba --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Services.py @@ -0,0 +1,25 @@ +import os,sys +services_dir=os.environment.get("SERIVCES_DIR","/home/reaktor/nag.services") +def add_services(nodes): + for k,v in nodes.items(): + n = nodes[k] + try: + with open("{0}/{1}".format(services_dir,k)) as f: + n["services"] = [] + for line in f.readlines(): + n["services"].append(line.strip()) + except Exception as e: + n["services"] = ["Error: No Service File!"] + return nodes +def main(): + import json,sys + try: + nodes = add_services(json.load(sys.stdin)) + print (json.dumps(nodes,indent=4)) + except: + print("unable to parse json data from stdin") + sys.exit(1) + +if __name__ == "__main__": + main() +# vim: set expandtab:ts=4:sw=4 diff --git a/retiolum/scripts/adv_graphgen/tinc_graphs/Supernodes.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Supernodes.py new file mode 100755 index 00000000..c8514c66 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Supernodes.py @@ -0,0 +1,65 @@ +#!/usr/bin/python3 + +def find_potential_super(path="/etc/tinc/retiolum/hosts"): + import os + import re + + needle_addr = re.compile("Address\s*=\s*(.*)") + needle_port = re.compile("Port\s*=\s*(.*)") + for f in os.listdir(path): + with open(path+"/"+f) as of: + addrs = [] + port = "655" + + for line in of.readlines(): + + addr_found = needle_addr.match(line) + if addr_found: + addrs.append(addr_found.group(1)) + + port_found = needle_port.match(line) + if port_found: + port = port_found.group(1) + + if addrs : yield (f ,[(addr ,int(port)) for addr in addrs]) + + +def try_connect(addr): + try: + from socket import socket,AF_INET,SOCK_STREAM + s = socket(AF_INET,SOCK_STREAM) + s.settimeout(2) + s.connect(addr) + s.settimeout(None) + s.close() + return addr + except Exception as e: + pass + + +def check_one_super(ha): + host,addrs = ha + valid_addrs = [] + for addr in addrs: + ret = try_connect(addr) + if ret: valid_addrs.append(ret) + if valid_addrs: return (host,valid_addrs) + + +def check_all_the_super(path): + from multiprocessing import Pool + p = Pool(20) + return filter(None,p.map(check_one_super,find_potential_super(path))) + + +def main(): + import os + hostpath=os.environment.get("TINC_HOSTPATH", "/etc/tinc/retiolum/hosts") + + for host,addrs in check_all_the_super(hostpath): + print("%s %s" %(host,str(addrs))) + +if __name__ == "__main__": + main() + +# vim: set expandtab:ts=:sw=2 diff --git a/retiolum/scripts/adv_graphgen/tinc_graphs/__init__.py b/retiolum/scripts/adv_graphgen/tinc_graphs/__init__.py new file mode 100644 index 00000000..414ffe99 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/__init__.py @@ -0,0 +1 @@ +__version__="0.2.3" diff --git a/retiolum/scripts/adv_graphgen/tinc_graphs/static/map.html b/retiolum/scripts/adv_graphgen/tinc_graphs/static/map.html new file mode 100644 index 00000000..ef8a0565 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_graphs/static/map.html @@ -0,0 +1,88 @@ + + + + + + Simple Map + + + + + + + + + +
+ + diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py b/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py deleted file mode 100755 index 66defa44..00000000 --- a/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf8 -*- - -import sys,json -""" TODO: Refactoring needed to pull the edges out of the node structures again, -it should be easier to handle both structures""" -DUMP_FILE = "/krebs/db/availability" - -def get_all_nodes(): - import os - return os.listdir("/etc/tinc/retiolum/hosts") - -def generate_stats(): - """ Generates availability statistics of the network and nodes - """ - import json - jlines = [] - try: - f = open(DUMP_FILE,'r') - for line in f: - jlines.append(json.loads(line)) - f.close() - except Exception as e: - pass - all_nodes = {} - for k in get_all_nodes(): - all_nodes[k] = get_node_availability(k,jlines) - print ( json.dumps(all_nodes)) - -def get_node_availability(name,jlines): - """ calculates the node availability by reading the generated dump file - adding together the uptime of the node and returning the time - parms: - name - node name - jlines - list of already parsed dictionaries node archive - """ - begin = last = current = 0 - uptime = 0 - for stat in jlines: - if not stat['nodes']: - continue - ts = stat['timestamp'] - if not begin: - begin = last = ts - current = ts - if stat['nodes'].get(name,{}).get('to',[]): - uptime += current - last - else: - pass - last = ts - all_the_time = last - begin - try: - return uptime/ all_the_time - except: - return 1 - -if __name__ == "__main__": - generate_stats() diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py b/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py deleted file mode 100644 index 6bdbf43c..00000000 --- a/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py +++ /dev/null @@ -1,35 +0,0 @@ -import sys -import os -import string - -class BackwardsReader: - """ Stripped and stolen from : http://code.activestate.com/recipes/120686-read-a-text-file-backwards/ """ - def readline(self): - while len(self.data) == 1 and ((self.blkcount * self.blksize) < self.size): - self.blkcount = self.blkcount + 1 - line = self.data[0] - try: - self.f.seek(-self.blksize * self.blkcount, 2) - self.data = string.split(self.f.read(self.blksize) + line, '\n') - except IOError: - self.f.seek(0) - self.data = string.split(self.f.read(self.size - (self.blksize * (self.blkcount-1))) + line, '\n') - - if len(self.data) == 0: - return "" - - line = self.data[-1] - self.data = self.data[:-1] - return line + '\n' - - def __init__(self, file, blksize=4096): - """initialize the internal structures""" - self.size = os.stat(file)[6] - self.blksize = blksize - self.blkcount = 1 - self.f = open(file, 'rb') - if self.size > self.blksize: - self.f.seek(-self.blksize * self.blkcount, 2) - self.data = string.split(self.f.read(self.blksize), '\n') - if not self.data[-1]: - self.data = self.data[:-1] diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py b/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py deleted file mode 100755 index 038ca9c0..00000000 --- a/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/python3 -# -*- coding: utf8 -*- -import sys,json,os -from Graph import delete_unused_nodes,resolve_myself -GEODB=os.environ.get("GEOCITYDB","GeoLiteCity.dat") - -def add_geo(nodes): - from pygeoip import GeoIP - gi = GeoIP(GEODB) - - for k,v in nodes.iteritems(): - try: - nodes[k].update(gi.record_by_addr(v["external-ip"])) - except Exception as e: - sys.stderr.write(str(e)) - sys.stderr.write("Cannot determine GeoData for %s\n"%k) - - return nodes -def add_coords_to_edges(nodes): - from pygeoip import GeoIP - gi = GeoIP(GEODB) - - for k,v in nodes.iteritems(): - for i,j in enumerate(v.get("to",[])): - data=gi.record_by_addr(j["addr"]) - try: - j["latitude"]=data["latitude"] - j["longitude"]=data["longitude"] - except Exception as e: pass - - return nodes - -def add_jitter(nodes): - from random import random - #add a bit of jitter to all of the coordinates - max_jitter=0.005 - for k,v in nodes.iteritems(): - jitter_lat= max_jitter -random()*max_jitter*2 - jitter_long= max_jitter -random()*max_jitter*2 - try: - v["latitude"]= v["latitude"] + jitter_lat - v["longitude"]= v["longitude"] + jitter_long - for nodek,node in nodes.iteritems(): - for to in node['to']: - if to['name'] == k: - to['latitude'] = v["latitude"] - to['longitude'] = v["longitude"] - except Exception as e: pass - return nodes - -if __name__ == "__main__": - import json - nodes = add_jitter(add_coords_to_edges(add_geo(resolve_myself(delete_unused_nodes(json.load(sys.stdin)))))) - print (json.dumps(nodes)) diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py b/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py deleted file mode 100755 index 2fb09a58..00000000 --- a/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py +++ /dev/null @@ -1,250 +0,0 @@ -#!/usr/bin/python -from BackwardsReader import BackwardsReader -import sys,json,os -from Supernodes import check_all_the_super -from Services import add_services -from Availability import get_node_availability -import sys,json -from time import time -DUMP_FILE = "/krebs/db/availability" - - -def resolve_myself(nodes): - #resolve MYSELF to the real ip - for k,v in nodes.items(): - if v["external-ip"] == "MYSELF": - for nodek,node in nodes.items(): - for to in node['to']: - if to['name'] == k: - v["external-ip"] = to["addr"] - return nodes - -def dump_graph(nodes): - from time import time - graph = {} - graph['nodes'] = nodes - graph['timestamp'] = time() - f = open(DUMP_FILE,'a') - json.dump(graph,f) - f.write('\n') - f.close() - -def generate_availability_stats(nodes): - """ generates stats of from availability - """ - jlines = [] - try: - f = BackwardsReader(DUMP_FILE) - lines_to_use = 1000 - while True: - if lines_to_use == 0: break - line = f.readline() - if not line: break - jline = json.loads(line) - if not jline['nodes']: continue - - jlines.append(jline) - lines_to_use -=1 - except Exception as e: sys.stderr.write(str(e)) - - for k,v in nodes.items(): - v['availability'] = get_node_availability(k,jlines) - sys.stderr.write( "%s -> %f\n" %(k ,v['availability'])) - -def generate_stats(nodes): - """ Generates some statistics of the network and nodes - """ - for k,v in nodes.items(): - conns = v.get('to',[]) - for c in conns: #sanitize weights - if float(c['weight']) > 9000: c['weight'] = str(9001) - elif float(c['weight']) < 0: c['weight'] = str(0) - v['num_conns'] = len(conns) - v['avg_weight'] = get_node_avg_weight(conns) - -def get_node_avg_weight(conns): - """ calculates the average weight for the given connections """ - if not conns: - sys.syderr.write("get_node_avg_weight: connection parameter empty") - return 9001 - else: - return sum([float(c['weight']) for c in conns])/len(conns) - -def delete_unused_nodes(nodes): - """ Deletes all the nodes which are currently not connected to the network""" - new_nodes = {} - for k,v in nodes.items(): - if v['external-ip'] == "(null)": - continue - if v.get('to',[]): - new_nodes[k] = v - for k,v in new_nodes.items(): - if not [ i for i in v['to'] if i['name'] in new_nodes]: - del(k) - return new_nodes - -def merge_edges(nodes): - """ merge back and forth edges into one - DESTRUCTS the current structure by deleting "connections" in the nodes - """ - for k,v in nodes.items(): - for con in v.get('to',[]): - for i,secon in enumerate(nodes.get(con['name'],{}).get('to',[])): - if k == secon['name']: - del (nodes[con['name']]['to'][i]) - con['bidirectional'] = True - - -def print_head(): - print ('digraph retiolum {') - print (' graph [center=true packMode="clust"]') - print (' node[shape=box,style=filled,fillcolor=grey]') - print (' overlap=false') - -def print_stat_node(nodes): - ''' Write a `stats` node in the corner - This node contains infos about the current number of active nodes and connections inside the network - ''' - from time import localtime,strftime - num_conns = 0 - num_nodes = len(nodes) - try: - msg = '%s.num_nodes %d %d\r\n' %(g_path,num_nodes,begin) - s.send(msg) - except Exception as e: pass - for k,v in nodes.items(): - num_conns+= len(v['to']) - node_text = " stats_node [label=\"Statistics\\l" - node_text += "Build Date : %s\\l" % strftime("%Y-%m-%d %H:%M:%S",localtime()) - node_text += "Active Nodes: %s\\l" % num_nodes - node_text += "Connections : %s\\l" % num_conns - node_text += "\"" - node_text += ",fillcolor=green" - node_text += "]" - print(node_text) - -def print_node(k,v): - """ writes a single node and its edges - edges are weightet with the informations inside the nodes provided by - tinc - """ - - node = " "+k+"[label=<" %k - node += ""%k - if 'availability' in v: - node += "" % v['availability'] - - if 'num_conns' in v: - node += ""%str(v['num_conns']) - - node += "" - for addr in v.get('internal-ip',['dunno lol']): - node += ""%addr - - if 'services' in v: - node +="" - for service in v['services']: - try:uri,comment = service.split(" ",1) - except: - uri = service - comment ="" - node +="" - uri_proto=uri.split(":")[0] - uri_rest = uri.split(":")[1] - if not uri_rest: - node +="" - # end label - node +="
%s
availability:%f
Num Connects:%s
external:"+v['external-ip']+":"+v['external-port']+"
internal:%s
Services:
{0}".format(uri) - else: - node +="{0}".format(uri) - if comment: - node += "
{0}".format(comment) - node +="
>" - - if v['num_conns'] == 1: - node += ",fillcolor=red" - elif k in supernodes: - node += ",fillcolor=steelblue1" - node += "]" - - print(node) - -def print_anonymous_node(k,v): - """ writes a single node and its edges - edges are weightet with the informations inside the nodes provided by - tinc - """ - - node = " "+k #+"[label=\"" - print(node) - -def print_edge(k,v): - for con in v.get('to',[]): - label = con['weight'] - w = int(con['weight']) - weight = str(1000 - (((w - 150) * (1000 - 0)) / (1000 -150 )) + 0) - - length = str(float(w)/1500) - if float(weight) < 0 : - weight= "1" - - edge = " "+k+ " -> " +con['name'] + " [label="+label + " weight="+weight - if con.get('bidirectional',False): - edge += ",dir=both" - edge += "]" - print(edge) - -def anonymize_nodes(nodes): - #anonymizes all nodes - i = "0" - newnodes = {} - for k,v in nodes.items(): - for nodek,node in nodes.items(): - for to in node['to']: - if to['name'] == k: - to['name'] = i - newnodes[i] = v - i = str(int(i)+1) - return newnodes - -if __name__ == "__main__": - supernodes= [] - if len(sys.argv) != 2 or sys.argv[1] not in ["anonymous","complete"]: - print("usage: %s (anonymous|complete)") - sys.exit(1) - - nodes = json.load(sys.stdin) - nodes = delete_unused_nodes(nodes) - print_head() - generate_stats(nodes) - merge_edges(nodes) - - - if sys.argv[1] == "anonymous": - nodes = anonymize_nodes(nodes) - - for k,v in nodes.items(): - print_anonymous_node(k,v) - print_edge(k,v) - - elif sys.argv[1] == "complete": - for supernode,addr in check_all_the_super(): - supernodes.append(supernode) - - generate_availability_stats(nodes) - add_services(nodes) - for k,v in nodes.items(): - print_node(k,v) - print_edge(k,v) - try: - dump_graph(nodes) - except Exception as e: - sys.stderr.write("Cannot dump graph: %s" % str(e)) - else: - pass - - print_stat_node(nodes) - print ('}') -# vim: set sw=2:ts=2 diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py b/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py deleted file mode 100755 index a81e2bef..00000000 --- a/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/python -import subprocess -import os -import re -import sys -import json - - -TINC_NETWORK =os.environ.get("TINC_NETWORK","retiolum") - -# is_legacy is the parameter which defines if the tinc config files are handled old fashioned (parse from syslog), -# or if the new and hip tincctl should be used - - -# Tags and Delimiters -TINC_TAG="tinc.%s" % TINC_NETWORK -BEGIN_NODES = "Nodes:" -END_NODES = "End of nodes." -BEGIN_SUBNET = "Subnet list:" -END_SUBNET = "End of subnet list" -BEGIN_EDGES = "Edges:" -END_EDGES = "End of edges." -def usage(): - from sys import argv,exit - print("""usage: %s -This tool dumps all tinc node informations as json - -ENVIRONMENT VARIABLES: - TINC_NETWORK The tinc network to dump - (default: retiolum) - LOG_FILE If legacy tinc is used, defines the log file where tinc stats are dumped in - (default: /var/log/everything.log) -""" % argv[0]) - exit(1) -def debug(func): - from functools import wraps - @wraps(func) - def with_debug(*args,**kwargs): - print( func.__name__ + " (args: %s | kwargs %s)"% (args,kwargs)) - return func(*args,**kwargs) - return with_debug - - -def parse_tinc_stats(): - import subprocess - from time import sleep - from distutils.spawn import find_executable as which - #newest tinc - if which("tinc"): - return parse_new_input("tinc") - #new tinc - elif which("tincctl"): - return parse_new_input("tincctl") - #old tinc - else: - raise Exception("no tinc executable found!") - -#@debug -def get_tinc_block(log_file): - """ returns an iterateable block from the given log file (syslog) - This function became obsolete with the introduction of tincctl - """ - from BackwardsReader import BackwardsReader - tinc_block = [] - in_block = False - bf = BackwardsReader(log_file) - BOL = re.compile(".*tinc.%s\[[0-9]+\]: " % TINC_NETWORK) - while True: - line = bf.readline() - if not line: - raise Exception("end of file at log file? This should not happen!") - line = BOL.sub('',line).strip() - - if END_SUBNET in line: - in_block = True - - if not in_block: - continue - tinc_block.append(line) - - if BEGIN_NODES in line: - break - return reversed(tinc_block) - -def parse_new_input(tinc_bin): - nodes = {} - pnodes = subprocess.Popen( - [tinc_bin,"-n",TINC_NETWORK,"dump","reachable","nodes"], - stdout=subprocess.PIPE).communicate()[0].decode() - #pnodes = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","reachable","nodes"]) - for line in pnodes.split('\n'): - if not line: continue - l = line.split() - nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] } - psubnets = subprocess.check_output( - [tinc_bin,"-n",TINC_NETWORK,"dump","subnets"]).decode() - for line in psubnets.split('\n'): - if not line: continue - l = line.split() - try: - if not nodes[l[2]].get('internal-ip',False): - nodes[l[2]]['internal-ip'] = [] - nodes[l[2]]['internal-ip'].append(l[0].split('#')[0]) - except KeyError: - pass # node does not exist (presumably) - pedges = subprocess.check_output( - [tinc_bin,"-n",TINC_NETWORK,"dump","edges"]).decode() - for line in pedges.split('\n'): - if not line: continue - l = line.split() - try: - if not 'to' in nodes[l[0]] : - nodes[l[0]]['to'] = [] - nodes[l[0]]['to'].append( - {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] }) - except KeyError: - pass #node does not exist - return nodes - -if __name__ == '__main__': - from sys import argv - if len(argv) > 1: - usage() - else: - print (json.dumps(parse_tinc_stats())) diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Services.py b/retiolum/scripts/adv_graphgen/tinc_stats/Services.py deleted file mode 100644 index 6752e116..00000000 --- a/retiolum/scripts/adv_graphgen/tinc_stats/Services.py +++ /dev/null @@ -1,17 +0,0 @@ -services_dir="/home/makefu/r/services" -def add_services(nodes): - for k,v in nodes.items(): - n = nodes[k] - try: - with open("{0}/{1}".format(services_dir,k)) as f: - n["services"] = [] - for line in f.readlines(): - n["services"].append(line.strip()) - except Exception as e: - n["services"] = ["Error: No Service File!"] - return nodes -if __name__ == "__main__": - import json,sys - nodes = add_services(json.load(sys.stdin)) - print (json.dumps(nodes,indent=4)) -# vim: set expandtab:ts=4:sw=4 diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py b/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py deleted file mode 100755 index 7e1f4dae..00000000 --- a/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/python3 - -def find_potential_super(path="/etc/tinc/retiolum/hosts"): - import os - import re - - needle_addr = re.compile("Address\s*=\s*(.*)") - needle_port = re.compile("Port\s*=\s*(.*)") - for f in os.listdir(path): - with open(path+"/"+f) as of: - addrs = [] - port = "655" - - for line in of.readlines(): - - addr_found = needle_addr.match(line) - if addr_found: - addrs.append(addr_found.group(1)) - - port_found = needle_port.match(line) - if port_found: - port = port_found.group(1) - - if addrs : yield (f ,[(addr ,int(port)) for addr in addrs]) - -def try_connect(addr): - try: - from socket import socket,AF_INET,SOCK_STREAM - s = socket(AF_INET,SOCK_STREAM) - s.settimeout(2) - s.connect(addr) - s.settimeout(None) - s.close() - return addr - except Exception as e: - pass - #return () - -def check_one_super(ha): - host,addrs = ha - valid_addrs = [] - for addr in addrs: - ret = try_connect(addr) - if ret: valid_addrs.append(ret) - if valid_addrs: return (host,valid_addrs) - -def check_all_the_super(path="/etc/tinc/retiolum/hosts"): - from multiprocessing import Pool - p = Pool(20) - return filter(None,p.map(check_one_super,find_potential_super(path))) - - - -if __name__ == "__main__": - """ - usage - """ - for host,addrs in check_all_the_super(): - print("%s %s" %(host,str(addrs))) diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py b/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py deleted file mode 100644 index f0fc8520..00000000 --- a/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -import Availability -import BackwardsReader -import Log2JSON -import Supernodes -import Geo -import Graph -import Services -- cgit v1.2.3