summaryrefslogtreecommitdiffstats
path: root/retiolum/scripts
diff options
context:
space:
mode:
authormakefu <github@syntax-fehler.de>2015-09-16 08:30:04 +0200
committermakefu <github@syntax-fehler.de>2015-09-16 08:30:04 +0200
commit147044be891c92b3c0c1bcc3a4e53e2d0eef9963 (patch)
tree13ed3ac36d514347f503043b454dddaee51b03ac /retiolum/scripts
parent9bf9f8d045801b17956d12f599bb166e608ed6dd (diff)
parent219fab970c7fe455d3dd9bc48e909d96a234046b (diff)
Merge branch 'master' of github.com:krebscode/painload
Diffstat (limited to 'retiolum/scripts')
-rw-r--r--retiolum/scripts/adv_graphgen/.gitignore5
-rw-r--r--retiolum/scripts/adv_graphgen/README28
-rw-r--r--retiolum/scripts/adv_graphgen/README.md38
-rwxr-xr-xretiolum/scripts/adv_graphgen/all_the_graphs.sh16
-rwxr-xr-xretiolum/scripts/adv_graphgen/anonytize.sh29
-rwxr-xr-xretiolum/scripts/adv_graphgen/sanitize.sh31
-rwxr-xr-xretiolum/scripts/adv_graphgen/scripts/all-the-graphs18
-rwxr-xr-xretiolum/scripts/adv_graphgen/scripts/build-graphs27
-rw-r--r--retiolum/scripts/adv_graphgen/setup.py42
-rw-r--r--retiolum/scripts/adv_graphgen/tinc_graphs.nix29
-rwxr-xr-xretiolum/scripts/adv_graphgen/tinc_graphs/Availability.py (renamed from retiolum/scripts/adv_graphgen/tinc_stats/Availability.py)17
-rw-r--r--retiolum/scripts/adv_graphgen/tinc_graphs/BackwardsReader.py (renamed from retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py)0
-rwxr-xr-xretiolum/scripts/adv_graphgen/tinc_graphs/Geo.py (renamed from retiolum/scripts/adv_graphgen/tinc_stats/Geo.py)44
-rwxr-xr-xretiolum/scripts/adv_graphgen/tinc_graphs/Graph.py (renamed from retiolum/scripts/adv_graphgen/tinc_stats/Graph.py)71
-rwxr-xr-xretiolum/scripts/adv_graphgen/tinc_graphs/Log2JSON.py (renamed from retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py)68
-rw-r--r--retiolum/scripts/adv_graphgen/tinc_graphs/Services.py (renamed from retiolum/scripts/adv_graphgen/tinc_stats/Services.py)16
-rwxr-xr-xretiolum/scripts/adv_graphgen/tinc_graphs/Supernodes.py (renamed from retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py)49
-rw-r--r--retiolum/scripts/adv_graphgen/tinc_graphs/__init__.py1
-rw-r--r--retiolum/scripts/adv_graphgen/tinc_graphs/static/map.html (renamed from retiolum/scripts/adv_graphgen/map.html)0
-rw-r--r--retiolum/scripts/adv_graphgen/tinc_stats/__init__.py7
20 files changed, 320 insertions, 216 deletions
diff --git a/retiolum/scripts/adv_graphgen/.gitignore b/retiolum/scripts/adv_graphgen/.gitignore
new file mode 100644
index 00000000..fbee25a5
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/.gitignore
@@ -0,0 +1,5 @@
+bin/
+include/
+lib/
+*.egg-info/
+GeoLiteCity.dat
diff --git a/retiolum/scripts/adv_graphgen/README b/retiolum/scripts/adv_graphgen/README
deleted file mode 100644
index 082e0f2b..00000000
--- a/retiolum/scripts/adv_graphgen/README
+++ /dev/null
@@ -1,28 +0,0 @@
-The folder contains a number of scripts which provide a convenient way to
-generate advanced graphs from the SIGUSR2 output of tinc.
-
-it currently contains the following files:
-
-sanitize.sh:
- wrapper arond parse.py which filters the syslog file for all tinc
- related lines and removes the status informations:
- this means that
- <code>
- May 19 20:40:44 servarch dnsmasq[5382]: reading /etc/resolv.conf
- May 19 20:41:38 servarch tinc.retiolum[4780]: Error looking up pa-sharepoint.informatik.ba-stuttgart.de port 655: Name or service not known
- </code>
- becomes
- <code>
- Error looking up pa-sharepoint.informatik.ba-stuttgart.de port 655: Name or service not known
- </code>
- and so on.
- It also provides a wrapper around graphviz which automagically
- generates graphs from the produced graph file
-
-parse.py:
- reads from stdin the sanitized syslog file and prints a valid dot file
- from the given output.
- The parser module may also produce any other output (e.g. for dns
- entries and so on) you will need to actually read and modify the source
- in order to be able to do this. ~May the source be with you~
-
diff --git a/retiolum/scripts/adv_graphgen/README.md b/retiolum/scripts/adv_graphgen/README.md
new file mode 100644
index 00000000..0f3ee285
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/README.md
@@ -0,0 +1,38 @@
+# Generate Graphs from tinc
+
+## Install
+### Nix
+
+ # tinc_pre is required:
+ nix-env -i -f tinc_graphs.nix
+
+ ## e.g. in Retiolum:
+ ## krebs.retiolum.tinc = pkgs.tinc_pre
+### Local
+
+ python setup.py install
+ # also install graphviz,imagemagic for building graphs
+
+
+### Usage:
+
+see source of the 2 builder scripts:
+
+ #all-around-builder
+ # env: EXTERNAL_FOLDER, INTERNAL_FOLDER, GEODB, TINC_HOSTPATH
+ all-the-graphs
+
+ # build actual graphs
+ build-graph
+
+ # exported py scripts
+ tinc-stats2json # - parses tinc current state into json
+ tinc-build-graph # - transfers json to graph
+ copy-map # - copies map.html into $1
+ add-geodata # - adds geodata to json
+ tinc-availability-stats # adds availability data to json
+
+## Geodb infos
+
+- http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz
+- nix-env -iA geolite-legacy
diff --git a/retiolum/scripts/adv_graphgen/all_the_graphs.sh b/retiolum/scripts/adv_graphgen/all_the_graphs.sh
deleted file mode 100755
index 9f3bf82b..00000000
--- a/retiolum/scripts/adv_graphgen/all_the_graphs.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/bash
-
-(
- echo "`date` begin all graphs" >> /tmp/build_graph
- cd $(dirname $(readlink -f $0))
- PATH=$PATH:../../../util/bin/
- EXTERNAL_FOLDER=/var/www/euer.krebsco.de/graphs/retiolum
- INTERNAL_FOLDER=/var/www/euer/graphs/retiolum
- begin=`timer`
- export GEOCTIYDB="$PWD/GeoLiteCity.dat"
- (python tinc_stats/Log2JSON.py | python tinc_stats/Geo.py > $INTERNAL_FOLDER/marker.json)&
- (./anonytize.sh $EXTERNAL_FOLDER && echo "`date` anonytize done" >> /tmp/build_graph)&
- (./sanitize.sh $INTERNAL_FOLDER && echo "`date` sanitize done" >> /tmp/build_graph)&
-# wait
- echo "`date` end all graphs" >> /tmp/build_graph
-)&
diff --git a/retiolum/scripts/adv_graphgen/anonytize.sh b/retiolum/scripts/adv_graphgen/anonytize.sh
deleted file mode 100755
index 04a68869..00000000
--- a/retiolum/scripts/adv_graphgen/anonytize.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/sh
-set -euf
-cd $(dirname `readlink -f $0`)
-GRAPH_SETTER1=dot
-GRAPH_SETTER2=circo
-GRAPH_SETTER3='neato -Goverlap=prism '
-GRAPH_SETTER4=sfdp
-TYPE=svg
-TYPE2=png
-OPENER=/bin/true
-DOTFILE=`mktemp --suffix=anon`
-trap 'rm $DOTFILE' INT TERM KILL EXIT
-sudo -E python tinc_stats/Log2JSON.py |\
- python tinc_stats/Graph.py anonymous> $DOTFILE
-
-
-i=1
-for setter in dot circo 'neato -Goverlap=prism ' sfdp
-do
- tmpgraph=`mktemp --tmpdir=$1`
- $setter -T$TYPE -o $tmpgraph $DOTFILE
- chmod go+rx $tmpgraph
- mv $tmpgraph $1/retiolum_$i.$TYPE
- i=`expr $i + 1`
-done
-convert $1/retiolum_1.$TYPE $1/retiolum_1.$TYPE2
-#convert -resize 20% $1/retiolum_2.$TYPE $1/retiolum_2.$TYPE2
-#convert -resize 20% $1/retiolum_3.$TYPE $1/retiolum_3.$TYPE2
-#convert -resize 20% $1/retiolum_4.$TYPE $1/retiolum_4.$TYPE2
diff --git a/retiolum/scripts/adv_graphgen/sanitize.sh b/retiolum/scripts/adv_graphgen/sanitize.sh
deleted file mode 100755
index f7d0e7e7..00000000
--- a/retiolum/scripts/adv_graphgen/sanitize.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/sh
-set -euf
-cd $(dirname `readlink -f $0`)
-GRAPH_SETTER1=dot
-GRAPH_SETTER2=circo
-GRAPH_SETTER3='neato -Goverlap=prism '
-GRAPH_SETTER4=sfdp
-TYPE=svg
-TYPE2=png
-OPENER=/bin/true
-DOTFILE=`mktemp --suffix=san`
-trap 'rm $DOTFILE' INT TERM KILL EXIT
-sudo -E python tinc_stats/Log2JSON.py |\
- python tinc_stats/Graph.py complete > $DOTFILE
-
-
-i=1
-for setter in dot circo 'neato -Goverlap=prism ' sfdp
-do
- tmpgraph=`mktemp --tmpdir=$1`
- $setter -T$TYPE -o $tmpgraph $DOTFILE
- chmod go+rx $tmpgraph
- mv $tmpgraph $1/retiolum_$i.$TYPE
- i=`expr $i + 1`
-done
-
-convert $1/retiolum_1.$TYPE $1/retiolum_1.$TYPE2
-#convert -resize 20% $1/retiolum_2.$TYPE $1/retiolum_2.$TYPE2
-#convert -resize 20% $1/retiolum_3.$TYPE $1/retiolum_3.$TYPE2
-#convert -resize 20% $1/retiolum_4.$TYPE $1/retiolum_4.$TYPE2
-rm $DOTFILE
diff --git a/retiolum/scripts/adv_graphgen/scripts/all-the-graphs b/retiolum/scripts/adv_graphgen/scripts/all-the-graphs
new file mode 100755
index 00000000..889a9fe0
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/scripts/all-the-graphs
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+EXTERNAL_FOLDER=${EXTERNAL_FOLDER:-/var/www/euer.krebsco.de/graphs/retiolum}
+INTERNAL_FOLDER=${INTERNAL_FOLDER:-/var/www/euer/graphs/retiolum}
+export GEODB="${GEODB:-}"
+export TINC_HOSTPATH=${TINC_HOSTPATH:-~/painload/retiolum/hosts}
+mapfile="$INTERNAL_FOLDER/map.html"
+
+if test -n "$GEOCITYDB";then
+ if test ! -e "$mapfile";then
+ echo "copying map to $map.html"
+ copy-map "$mapfile"
+ fi
+ echo "creating geodata database"
+ tinc-stats2json | add-geodata > "$INTERNAL_FOLDER/marker.json"
+fi
+build-graphs anonymous "$EXTERNAL_FOLDER"
+build-graphs complete "$INTERNAL_FOLDER"
diff --git a/retiolum/scripts/adv_graphgen/scripts/build-graphs b/retiolum/scripts/adv_graphgen/scripts/build-graphs
new file mode 100755
index 00000000..3eb675ec
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/scripts/build-graphs
@@ -0,0 +1,27 @@
+#!/bin/sh
+set -euf
+mode=${1?arg1 must be either 'anonymous' or 'complete'}
+out=${2?arg2 must be set to an outdir}
+# for creating all the graphs
+TYPE=svg
+# for creating real pictures
+TYPE2=png
+
+DOTFILE=`mktemp`
+trap 'rm "$DOTFILE"' INT TERM KILL EXIT
+
+tinc-stats2json | tinc-build-graph "$mode" > "$DOTFILE"
+
+i=1
+for setter in dot circo 'neato -Goverlap=prism ' sfdp
+do
+ tmpgraph=`mktemp --tmpdir=$1`
+ # first build, then move to avoid half-built graphs
+ "$setter" -T$TYPE -o "$tmpgraph" "$DOTFILE" ||:
+ chmod go+rx "$tmpgraph"
+ mv "$tmpgraph" "$1/retiolum_$i.$TYPE"
+ i=`expr $i + 1`
+done
+
+convert -limit memory 32Mib -limit map 64Mib "$out/retiolum_1.$TYPE" "$out/retiolum_1.$TYPE2"
+rm "$DOTFILE"
diff --git a/retiolum/scripts/adv_graphgen/setup.py b/retiolum/scripts/adv_graphgen/setup.py
new file mode 100644
index 00000000..dd772ef9
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/setup.py
@@ -0,0 +1,42 @@
+import sys
+from setuptools import setup
+
+setup(
+ name='tinc_graphs',
+ version='0.2.3',
+
+ description='Create Graphs from tinc Stats',
+ long_description=open("README.md").read(),
+ license='WTFPL',
+ url='http://krebsco.de/',
+ download_url='https://pypi.python.org/pypi/tinc_graphs/',
+
+ author='krebs',
+ author_email='spam@krebsco.de',
+ # you will also need graphviz and imagemagick
+ install_requires = [ 'pygeoip' ],
+ scripts = ['scripts/all-the-graphs', 'scripts/build-graphs'],
+ packages=['tinc_graphs'],
+ entry_points={
+ 'console_scripts' : [
+ 'tinc-stats2json = tinc_graphs.Log2JSON:main',
+ 'tinc-build-graph = tinc_graphs.Graph:main',
+ 'copy-map = tinc_graphs.Geo:copy_map',
+ 'add-geodata = tinc_graphs.Geo:main',
+ 'tinc-availability-stats = tinc_graphs.Availability:generate_stats',
+ ]
+ },
+
+ classifiers=[
+ "Intended Audience :: Developers",
+ "Natural Language :: English",
+ "Operating System :: POSIX :: Linux",
+ "Development Status :: 3 - Alpha",
+ "Programming Language :: Python",
+ "Programming Language :: Python :: 2.7",
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3.4",
+ "Programming Language :: Python :: Implementation :: CPython",
+ ],
+)
+
diff --git a/retiolum/scripts/adv_graphgen/tinc_graphs.nix b/retiolum/scripts/adv_graphgen/tinc_graphs.nix
new file mode 100644
index 00000000..662e81ac
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/tinc_graphs.nix
@@ -0,0 +1,29 @@
+with import <nixpkgs> {};
+# nix-build Reaktor.nix
+# result/bin/reaktor
+## or in your env
+# nix-env -i -f tinc_graphs.nix
+
+python3Packages.buildPythonPackage rec {
+ name = "tinc_graphs-${version}";
+ version = "0.2.6";
+ propagatedBuildInputs = with pkgs;[
+ graphviz
+ imagemagick
+
+ # optional if you want geolocation:
+ python3Packages.pygeoip
+ # geolite-legacy for the db:
+ ## ${geolite-legacy}/share/GeoIP/GeoIPCity.dat
+ ];
+ #src = fetchurl {
+ #url = "";
+ #sha256 = "1dksw1s1n2hxvnga6pygkr174dywncr0wiggkrkn1srbn2amh1c2";
+ #};
+ src = ./.;
+ meta = {
+ homepage = http://krebsco.de/;
+ description = "Create Graphs from Tinc Stats";
+ license = stdenv.lib.licenses.wtfpl;
+ };
+}
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Availability.py
index 66defa44..888335a7 100755
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Availability.py
+++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Availability.py
@@ -1,14 +1,15 @@
#!/usr/bin/python
+# TODO: Rewrite this shitty piece of software ...
# -*- coding: utf8 -*-
-import sys,json
+import sys,json,os
""" TODO: Refactoring needed to pull the edges out of the node structures again,
it should be easier to handle both structures"""
-DUMP_FILE = "/krebs/db/availability"
+DUMP_FILE = os.environ.get("AVAILABILITY_FILE","tinc-availability.json")
+hostpath=os.environ.get("TINC_HOSTPATH", "/etc/tinc/retiolum/hosts")
def get_all_nodes():
- import os
- return os.listdir("/etc/tinc/retiolum/hosts")
+ return os.listdir(hostpath)
def generate_stats():
""" Generates availability statistics of the network and nodes
@@ -16,16 +17,18 @@ def generate_stats():
import json
jlines = []
try:
- f = open(DUMP_FILE,'r')
+ f = open(DUMP_FILE,'r+')
for line in f:
jlines.append(json.loads(line))
f.close()
except Exception as e:
- pass
+ print("Unable to open and parse Availability DB: {} (override with AVAILABILITY_FILE)".format(DUMP_FILE))
+ sys.exit(1)
+
all_nodes = {}
for k in get_all_nodes():
all_nodes[k] = get_node_availability(k,jlines)
- print ( json.dumps(all_nodes))
+ print (json.dumps(all_nodes))
def get_node_availability(name,jlines):
""" calculates the node availability by reading the generated dump file
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py b/retiolum/scripts/adv_graphgen/tinc_graphs/BackwardsReader.py
index 6bdbf43c..6bdbf43c 100644
--- a/retiolum/scripts/adv_graphgen/tinc_stats/BackwardsReader.py
+++ b/retiolum/scripts/adv_graphgen/tinc_graphs/BackwardsReader.py
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Geo.py
index 038ca9c0..bfa4ee56 100755
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py
+++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Geo.py
@@ -1,26 +1,37 @@
#!/usr/bin/python3
# -*- coding: utf8 -*-
import sys,json,os
-from Graph import delete_unused_nodes,resolve_myself
-GEODB=os.environ.get("GEOCITYDB","GeoLiteCity.dat")
+from .Graph import delete_unused_nodes,resolve_myself
+GEODB=os.environ.get("GEODB","GeoLiteCity.dat")
+
+def copy_map():
+ from shutil import copy
+ from os.path import dirname,join,realpath
+ if len(sys.argv) != 2 or sys.argv[1] == "--help" :
+ print("usage: {} <destination>".format(sys.argv[0]))
+ print(" copies the map.html file to the <destination>")
+ sys.exit(1)
+ dstdir=sys.argv[1]
+ copy(realpath(join(dirname(__file__),'static/map.html')),dstdir)
+
def add_geo(nodes):
from pygeoip import GeoIP
gi = GeoIP(GEODB)
- for k,v in nodes.iteritems():
+ for k,v in nodes.items():
try:
nodes[k].update(gi.record_by_addr(v["external-ip"]))
except Exception as e:
sys.stderr.write(str(e))
sys.stderr.write("Cannot determine GeoData for %s\n"%k)
-
return nodes
+
def add_coords_to_edges(nodes):
from pygeoip import GeoIP
gi = GeoIP(GEODB)
- for k,v in nodes.iteritems():
+ for k,v in nodes.items():
for i,j in enumerate(v.get("to",[])):
data=gi.record_by_addr(j["addr"])
try:
@@ -34,13 +45,13 @@ def add_jitter(nodes):
from random import random
#add a bit of jitter to all of the coordinates
max_jitter=0.005
- for k,v in nodes.iteritems():
+ for k,v in nodes.items():
jitter_lat= max_jitter -random()*max_jitter*2
jitter_long= max_jitter -random()*max_jitter*2
try:
v["latitude"]= v["latitude"] + jitter_lat
v["longitude"]= v["longitude"] + jitter_long
- for nodek,node in nodes.iteritems():
+ for nodek,node in nodes.items():
for to in node['to']:
if to['name'] == k:
to['latitude'] = v["latitude"]
@@ -48,7 +59,20 @@ def add_jitter(nodes):
except Exception as e: pass
return nodes
-if __name__ == "__main__":
+def main():
import json
- nodes = add_jitter(add_coords_to_edges(add_geo(resolve_myself(delete_unused_nodes(json.load(sys.stdin))))))
- print (json.dumps(nodes))
+ try:
+ with open(GEODB,'rb') as f: f.read()
+ except Exception as e:
+ print("cannot open {} (GEODB in env)".format(GEODB))
+ print(e)
+ sys.exit(1)
+ try:
+ nodes = add_jitter(add_coords_to_edges(add_geo(resolve_myself(delete_unused_nodes(json.load(sys.stdin))))))
+ print (json.dumps(nodes))
+ except Exception as e:
+ print("cannot parse data received via stdin")
+ print(e)
+
+if __name__ == "__main__":
+ main()
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Graph.py
index 2fb09a58..29491997 100755
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py
+++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Graph.py
@@ -1,13 +1,16 @@
#!/usr/bin/python
-from BackwardsReader import BackwardsReader
+from .BackwardsReader import BackwardsReader
import sys,json,os
-from Supernodes import check_all_the_super
-from Services import add_services
-from Availability import get_node_availability
+from .Supernodes import check_all_the_super
+from .Services import add_services
+from .Availability import get_node_availability
import sys,json
from time import time
-DUMP_FILE = "/krebs/db/availability"
+DUMP_FILE = os.environ.get("AVAILABILITY_FILE", "tinc-availability.json")
+hostpath=os.environ.get("TINC_HOSTPATH", "/etc/tinc/retiolum/hosts")
+# will be filled later
+supernodes= []
def resolve_myself(nodes):
#resolve MYSELF to the real ip
@@ -19,6 +22,7 @@ def resolve_myself(nodes):
v["external-ip"] = to["addr"]
return nodes
+
def dump_graph(nodes):
from time import time
graph = {}
@@ -33,23 +37,24 @@ def generate_availability_stats(nodes):
""" generates stats of from availability
"""
jlines = []
- try:
- f = BackwardsReader(DUMP_FILE)
- lines_to_use = 1000
- while True:
- if lines_to_use == 0: break
- line = f.readline()
- if not line: break
- jline = json.loads(line)
- if not jline['nodes']: continue
-
- jlines.append(jline)
- lines_to_use -=1
- except Exception as e: sys.stderr.write(str(e))
+ # try:
+ # f = BackwardsReader(DUMP_FILE)
+ # lines_to_use = 1000
+ # while True:
+ # if lines_to_use == 0: break
+ # line = f.readline()
+ # if not line: break
+ # jline = json.loads(line)
+ # if not jline['nodes']: continue
+
+ # jlines.append(jline)
+ # lines_to_use -=1
+ # except Exception as e: sys.stderr.write(str(e))
for k,v in nodes.items():
- v['availability'] = get_node_availability(k,jlines)
- sys.stderr.write( "%s -> %f\n" %(k ,v['availability']))
+ # TODO: get this information in a different way
+ v['availability'] = get_node_availability(k,[])
+
def generate_stats(nodes):
""" Generates some statistics of the network and nodes
@@ -209,8 +214,7 @@ def anonymize_nodes(nodes):
i = str(int(i)+1)
return newnodes
-if __name__ == "__main__":
- supernodes= []
+def main():
if len(sys.argv) != 2 or sys.argv[1] not in ["anonymous","complete"]:
print("usage: %s (anonymous|complete)")
sys.exit(1)
@@ -230,21 +234,32 @@ if __name__ == "__main__":
print_edge(k,v)
elif sys.argv[1] == "complete":
- for supernode,addr in check_all_the_super():
- supernodes.append(supernode)
+ try:
+ for supernode,addr in check_all_the_super(hostpath):
+ supernodes.append(supernode)
+ except FileNotFoundError as e:
+ print("!! cannot load list of supernodes ({})".format(hostpath))
+ print("!! Use TINC_HOSTPATH env to override")
+ sys.exit(1)
generate_availability_stats(nodes)
add_services(nodes)
for k,v in nodes.items():
print_node(k,v)
print_edge(k,v)
- try:
- dump_graph(nodes)
- except Exception as e:
- sys.stderr.write("Cannot dump graph: %s" % str(e))
+
+ #TODO: get availability somehow else
+ # try:
+ # dump_graph(nodes)
+ # except Exception as e:
+ # sys.stderr.write("Cannot dump graph: %s" % str(e))
else:
pass
print_stat_node(nodes)
print ('}')
+
+if __name__ == "__main__":
+ main()
+
# vim: set sw=2:ts=2
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Log2JSON.py
index a81e2bef..b0bc209b 100755
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Log2JSON.py
+++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Log2JSON.py
@@ -26,12 +26,11 @@ def usage():
This tool dumps all tinc node informations as json
ENVIRONMENT VARIABLES:
- TINC_NETWORK The tinc network to dump
+ TINC_NETWORK The tinc network to dump
(default: retiolum)
- LOG_FILE If legacy tinc is used, defines the log file where tinc stats are dumped in
- (default: /var/log/everything.log)
""" % argv[0])
exit(1)
+
def debug(func):
from functools import wraps
@wraps(func)
@@ -52,46 +51,35 @@ def parse_tinc_stats():
elif which("tincctl"):
return parse_new_input("tincctl")
#old tinc
+ elif which("tincd"):
+ print("old tincd not supported")
+ sys.exit(1)
else:
raise Exception("no tinc executable found!")
-
-#@debug
-def get_tinc_block(log_file):
- """ returns an iterateable block from the given log file (syslog)
- This function became obsolete with the introduction of tincctl
- """
- from BackwardsReader import BackwardsReader
- tinc_block = []
- in_block = False
- bf = BackwardsReader(log_file)
- BOL = re.compile(".*tinc.%s\[[0-9]+\]: " % TINC_NETWORK)
- while True:
- line = bf.readline()
- if not line:
- raise Exception("end of file at log file? This should not happen!")
- line = BOL.sub('',line).strip()
-
- if END_SUBNET in line:
- in_block = True
-
- if not in_block:
- continue
- tinc_block.append(line)
-
- if BEGIN_NODES in line:
- break
- return reversed(tinc_block)
+
def parse_new_input(tinc_bin):
- nodes = {}
+ nodes = {}
pnodes = subprocess.Popen(
[tinc_bin,"-n",TINC_NETWORK,"dump","reachable","nodes"],
stdout=subprocess.PIPE).communicate()[0].decode()
- #pnodes = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","reachable","nodes"])
for line in pnodes.split('\n'):
if not line: continue
l = line.split()
- nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] }
+ n = l[0]
+
+ token = l[1]
+ if token == 'id':
+ # new format
+ # <name> id <ident> at <ip> port <port>
+ ident = l[1]
+ l = l[2:] #shift over 'id <ident>'
+ # else: # token = 'at'
+ # old format:
+ # <name> at <ip> port <port>
+ _,_,ip,_,port = l[:5]
+ nodes[n]= { 'external-ip': ip, 'external-port' : l[4] }
+
psubnets = subprocess.check_output(
[tinc_bin,"-n",TINC_NETWORK,"dump","subnets"]).decode()
for line in psubnets.split('\n'):
@@ -103,23 +91,29 @@ def parse_new_input(tinc_bin):
nodes[l[2]]['internal-ip'].append(l[0].split('#')[0])
except KeyError:
pass # node does not exist (presumably)
+
pedges = subprocess.check_output(
[tinc_bin,"-n",TINC_NETWORK,"dump","edges"]).decode()
for line in pedges.split('\n'):
if not line: continue
l = line.split()
+ # TODO: tokenize this and parse the line
+ n = l[0]
try:
if not 'to' in nodes[l[0]] :
- nodes[l[0]]['to'] = []
- nodes[l[0]]['to'].append(
- {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] })
+ nodes[n]['to'] = []
+ nodes[n]['to'].append(
+ {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[-1] })
except KeyError:
pass #node does not exist
return nodes
-if __name__ == '__main__':
+def main():
from sys import argv
if len(argv) > 1:
usage()
else:
print (json.dumps(parse_tinc_stats()))
+
+if __name__ == '__main__':
+ main()
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Services.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Services.py
index 6752e116..9581e21e 100644
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Services.py
+++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Services.py
@@ -1,4 +1,5 @@
-services_dir="/home/makefu/r/services"
+import os,sys
+services_dir=os.environ.get("SERIVCES_DIR","/home/reaktor/nag.services")
def add_services(nodes):
for k,v in nodes.items():
n = nodes[k]
@@ -10,8 +11,15 @@ def add_services(nodes):
except Exception as e:
n["services"] = ["Error: No Service File!"]
return nodes
-if __name__ == "__main__":
+def main():
import json,sys
- nodes = add_services(json.load(sys.stdin))
- print (json.dumps(nodes,indent=4))
+ try:
+ nodes = add_services(json.load(sys.stdin))
+ print (json.dumps(nodes,indent=4))
+ except:
+ print("unable to parse json data from stdin")
+ sys.exit(1)
+
+if __name__ == "__main__":
+ main()
# vim: set expandtab:ts=4:sw=4
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py b/retiolum/scripts/adv_graphgen/tinc_graphs/Supernodes.py
index 7e1f4dae..bc66b337 100755
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Supernodes.py
+++ b/retiolum/scripts/adv_graphgen/tinc_graphs/Supernodes.py
@@ -7,21 +7,27 @@ def find_potential_super(path="/etc/tinc/retiolum/hosts"):
needle_addr = re.compile("Address\s*=\s*(.*)")
needle_port = re.compile("Port\s*=\s*(.*)")
for f in os.listdir(path):
- with open(path+"/"+f) as of:
- addrs = []
- port = "655"
+ try:
+ with open(path+"/"+f) as of:
+ addrs = []
+ port = "655"
- for line in of.readlines():
+ for line in of.readlines():
- addr_found = needle_addr.match(line)
- if addr_found:
- addrs.append(addr_found.group(1))
+ addr_found = needle_addr.match(line)
+ if addr_found:
+ addrs.append(addr_found.group(1))
+
+ port_found = needle_port.match(line)
+ if port_found:
+ port = port_found.group(1)
+
+ if addrs : yield (f ,[(addr ,int(port)) for addr in addrs])
+ except FileNotFoundError as e:
+ print("Cannot open hosts directory to be used to find potential supernodes")
+ print("Directory used: {}".format(path))
+ raise
- port_found = needle_port.match(line)
- if port_found:
- port = port_found.group(1)
-
- if addrs : yield (f ,[(addr ,int(port)) for addr in addrs])
def try_connect(addr):
try:
@@ -34,7 +40,7 @@ def try_connect(addr):
return addr
except Exception as e:
pass
- #return ()
+
def check_one_super(ha):
host,addrs = ha
@@ -44,16 +50,21 @@ def check_one_super(ha):
if ret: valid_addrs.append(ret)
if valid_addrs: return (host,valid_addrs)
-def check_all_the_super(path="/etc/tinc/retiolum/hosts"):
+
+def check_all_the_super(path):
from multiprocessing import Pool
p = Pool(20)
return filter(None,p.map(check_one_super,find_potential_super(path)))
+def main():
+ import os
+ hostpath=os.environ.get("TINC_HOSTPATH", "/etc/tinc/retiolum/hosts")
-if __name__ == "__main__":
- """
- usage
- """
- for host,addrs in check_all_the_super():
+ for host,addrs in check_all_the_super(hostpath):
print("%s %s" %(host,str(addrs)))
+
+if __name__ == "__main__":
+ main()
+
+# vim: set expandtab:ts=:sw=2
diff --git a/retiolum/scripts/adv_graphgen/tinc_graphs/__init__.py b/retiolum/scripts/adv_graphgen/tinc_graphs/__init__.py
new file mode 100644
index 00000000..414ffe99
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/tinc_graphs/__init__.py
@@ -0,0 +1 @@
+__version__="0.2.3"
diff --git a/retiolum/scripts/adv_graphgen/map.html b/retiolum/scripts/adv_graphgen/tinc_graphs/static/map.html
index ef8a0565..ef8a0565 100644
--- a/retiolum/scripts/adv_graphgen/map.html
+++ b/retiolum/scripts/adv_graphgen/tinc_graphs/static/map.html
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py b/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py
deleted file mode 100644
index f0fc8520..00000000
--- a/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import Availability
-import BackwardsReader
-import Log2JSON
-import Supernodes
-import Geo
-import Graph
-import Services