diff options
| -rwxr-xr-x | retiolum/bin/create-host-tar | 16 | ||||
| -rwxr-xr-x | retiolum/bin/create-supernode-tar | 13 | ||||
| l--------- | retiolum/bin/tinc_stats | 1 | ||||
| l---------[-rwxr-xr-x] | retiolum/bin/tinc_stats2json | 124 | ||||
| -rwxr-xr-x | retiolum/scripts/adv_graphgen/all_the_graphs.sh | 6 | ||||
| -rwxr-xr-x | retiolum/scripts/adv_graphgen/tinc_stats2json | 124 | ||||
| -rw-r--r-- | retiolum/scripts/github_listener/github_listener.conf | 5 | ||||
| -rwxr-xr-x | retiolum/scripts/github_listener/handle_request | 19 | ||||
| -rwxr-xr-x | retiolum/scripts/github_listener/listen | 8 |
9 files changed, 169 insertions, 147 deletions
diff --git a/retiolum/bin/create-host-tar b/retiolum/bin/create-host-tar index 9def7cec..48ce26d5 100755 --- a/retiolum/bin/create-host-tar +++ b/retiolum/bin/create-host-tar @@ -1,9 +1,15 @@ #!/bin/sh -set -xeuf -WEBDIR=${WEBDIR:-/srv/http/pub/retiolum} +set -euf +if [ x"${1:-}" = "x" ];then + echo "usage: $0 DIRECTORY" + exit 1 +fi + +WEBDIR=$1 +HOSTDIR=${HOSTDIR:-../hosts} HOSTFILE=$WEBDIR/hosts.tar.gz + cd $(dirname $(readlink -f $0)) -git pull origin master || logger "cannot pull painload" -#rm $HOSTFILE -tar czf $HOSTFILE -C ../hosts . +rm -f $HOSTFILE +tar czf $HOSTFILE -C $HOSTDIR . chmod a+rx $HOSTFILE diff --git a/retiolum/bin/create-supernode-tar b/retiolum/bin/create-supernode-tar new file mode 100755 index 00000000..322d428d --- /dev/null +++ b/retiolum/bin/create-supernode-tar @@ -0,0 +1,13 @@ +#!/bin/sh +set -euf +if [ x"${1:-}" = "x" ];then + echo "usage: $0 DIRECTORY" + exit 1 +fi +HOSTDIR=${HOSTDIR:-../hosts} +WEBDIR=$1 +HOSTFILE=$WEBDIR/supernodes.tar.gz +cd $(dirname $(readlink -f $0)) +rm -f $HOSTFILE +tar czf $HOSTFILE -C $HOSTDIR `python ../scripts/adv_graphgen/find_super.py |cut -d\ -f1` +chmod a+rx $HOSTFILE diff --git a/retiolum/bin/tinc_stats b/retiolum/bin/tinc_stats deleted file mode 120000 index 6a58af60..00000000 --- a/retiolum/bin/tinc_stats +++ /dev/null @@ -1 +0,0 @@ -/home/makefu/repos/krebs/retiolum/scripts/adv_graphgen/tinc_stats.py
\ No newline at end of file diff --git a/retiolum/bin/tinc_stats2json b/retiolum/bin/tinc_stats2json index acadb306..413b778c 100755..120000 --- a/retiolum/bin/tinc_stats2json +++ b/retiolum/bin/tinc_stats2json @@ -1,123 +1 @@ -#!/usr/bin/python -import subprocess -import os -import re -import sys -import json - - - -TINC_NETWORK =os.environ.get("TINC_NETWORK","retiolum") - -# is_legacy is the parameter which defines if the tinc config files are handled old fashioned (parse from syslog), -# or if the new and hip tincctl should be used -is_legacy= os.environ.get("TINC_LEGACY",False) -SYSLOG_FILE = os.environ.get("LOG_FILE","/var/log/everything.log") - - -# Tags and Delimiters -TINC_TAG="tinc.%s" % TINC_NETWORK -BEGIN_NODES = "Nodes:" -END_NODES = "End of nodes." -BEGIN_SUBNET = "Subnet list:" -END_SUBNET = "End of subnet list" -BEGIN_EDGES = "Edges:" -END_EDGES = "End of edges." - -def get_tinc_block(log_file): - """ returns an iterateable block from the given log file (syslog) - This function became obsolete with the introduction of tincctl - """ - from BackwardsReader import BackwardsReader - tinc_block = [] - in_block = False - bf = BackwardsReader(log_file) - BOL = re.compile(".*tinc.%s\[[0-9]+\]: " % TINC_NETWORK) - while True: - line = bf.readline() - if not line: - raise Exception("end of file at log file? This should not happen!") - line = BOL.sub('',line).strip() - - if END_SUBNET in line: - in_block = True - - if not in_block: - continue - - tinc_block.append(line) - - if BEGIN_NODES in line: - break - return reversed(tinc_block) - -def parse_new_input(): - nodes = {} - pnodes = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","reachable","nodes"]) - for line in pnodes.split('\n'): - if not line: continue - l = line.split() - nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] } - psubnets = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","subnets"]) - for line in psubnets.split('\n'): - if not line: continue - l = line.split() - try: - if not nodes[l[2]].get('internal-ip',False): - nodes[l[2]]['internal-ip'] = [] - nodes[l[2]]['internal-ip'].append(l[0].split('#')[0]) - except KeyError: - pass # node does not exist (presumably) - pedges = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","edges"]) - for line in pedges.split('\n'): - if not line: continue - l = line.split() - try: - if not nodes[l[0]].has_key('to') : - nodes[l[0]]['to'] = [] - nodes[l[0]]['to'].append( - {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] }) - except KeyError: - pass #node does not exist - return nodes - -def parse_input(log_data): - nodes={} - for line in log_data: - if BEGIN_NODES in line : - nodes={} - for line in log_data: - if END_NODES in line : - break - l = line.replace('\n','').split() #TODO unhack me - nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] } - if BEGIN_SUBNET in line : - for line in log_data: - if END_SUBNET in line : - break - l = line.replace('\n','').split() - if not nodes[l[2]].get('internal-ip',False): - nodes[l[2]]['internal-ip'] = [] - nodes[l[2]]['internal-ip'].append(l[0].split('#')[0]) - if BEGIN_EDGES in line : - edges = {} - for line in log_data: - if END_EDGES in line : - break - l = line.replace('\n','').split() - if not nodes[l[0]].has_key('to') : - nodes[l[0]]['to'] = [] - nodes[l[0]]['to'].append( - {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] }) - return nodes - - -if __name__ == '__main__': - import subprocess,time - if is_legacy: - subprocess.call(["pkill","-SIGUSR2", "tincd"]) - time.sleep(1) - print json.dumps(parse_input((get_tinc_block(SYSLOG_FILE)))) - else: - print json.dumps(parse_new_input()) - +../scripts/adv_graphgen/tinc_stats2json
\ No newline at end of file diff --git a/retiolum/scripts/adv_graphgen/all_the_graphs.sh b/retiolum/scripts/adv_graphgen/all_the_graphs.sh index 5533c722..8aa0ab21 100755 --- a/retiolum/scripts/adv_graphgen/all_the_graphs.sh +++ b/retiolum/scripts/adv_graphgen/all_the_graphs.sh @@ -5,10 +5,12 @@ cd $(dirname $(readlink -f $0)) PATH=$PATH:../../../util/bin/ export LOG_FILE=/var/log/retiolum.log + EXTERNAL_FOLDER=/var/www/euer.krebsco.de/graphs/retiolum + INTERNAL_FOLDER=/var/www/euer/graphs/retiolum begin=`timer` export GRAPHITE_HOST="no_omo" - (./anonytize.sh /srv/http/pub/graphs/retiolum/ && echo "`date` anonytize done" >> /tmp/build_graph)& - (./sanitize.sh /srv/http/priv/graphs/retiolum/ && echo "`date` sanitize done" >> /tmp/build_graph)& + (./anonytize.sh $EXTERNAL_FOLDER && echo "`date` anonytize done" >> /tmp/build_graph)& + (./sanitize.sh $INTERNAL_FOLDER && echo "`date` sanitize done" >> /tmp/build_graph)& # wait graphitec "retiolum.graph.buildtime" "$(timer $begin)" >> /tmp/build_graph echo "`date` end all graphs" >> /tmp/build_graph diff --git a/retiolum/scripts/adv_graphgen/tinc_stats2json b/retiolum/scripts/adv_graphgen/tinc_stats2json new file mode 100755 index 00000000..ede19b26 --- /dev/null +++ b/retiolum/scripts/adv_graphgen/tinc_stats2json @@ -0,0 +1,124 @@ +#!/usr/bin/python +import subprocess +import os +import re +import sys +import json + + + +TINC_NETWORK =os.environ.get("TINC_NETWORK","retiolum") + +# is_legacy is the parameter which defines if the tinc config files are handled old fashioned (parse from syslog), +# or if the new and hip tincctl should be used +is_legacy= os.environ.get("TINC_LEGACY",False) +SYSLOG_FILE = os.environ.get("LOG_FILE","/var/log/everything.log") + + +# Tags and Delimiters +TINC_TAG="tinc.%s" % TINC_NETWORK +BEGIN_NODES = "Nodes:" +END_NODES = "End of nodes." +BEGIN_SUBNET = "Subnet list:" +END_SUBNET = "End of subnet list" +BEGIN_EDGES = "Edges:" +END_EDGES = "End of edges." + +def get_tinc_block(log_file): + """ returns an iterateable block from the given log file (syslog) + This function became obsolete with the introduction of tincctl + """ + from BackwardsReader import BackwardsReader + tinc_block = [] + in_block = False + bf = BackwardsReader(log_file) + BOL = re.compile(".*tinc.%s\[[0-9]+\]: " % TINC_NETWORK) + while True: + line = bf.readline() + if not line: + raise Exception("end of file at log file? This should not happen!") + line = BOL.sub('',line).strip() + + if END_SUBNET in line: + in_block = True + + if not in_block: + continue + + tinc_block.append(line) + + if BEGIN_NODES in line: + break + return reversed(tinc_block) + +def parse_new_input(): + nodes = {} + pnodes = subprocess.Popen(["tincctl","-n",TINC_NETWORK,"dump","reachable","nodes"], stdout=subprocess.PIPE).communicate()[0] + #pnodes = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","reachable","nodes"]) + for line in pnodes.split('\n'): + if not line: continue + l = line.split() + nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] } + psubnets = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","subnets"]) + for line in psubnets.split('\n'): + if not line: continue + l = line.split() + try: + if not nodes[l[2]].get('internal-ip',False): + nodes[l[2]]['internal-ip'] = [] + nodes[l[2]]['internal-ip'].append(l[0].split('#')[0]) + except KeyError: + pass # node does not exist (presumably) + pedges = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","edges"]) + for line in pedges.split('\n'): + if not line: continue + l = line.split() + try: + if not nodes[l[0]].has_key('to') : + nodes[l[0]]['to'] = [] + nodes[l[0]]['to'].append( + {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] }) + except KeyError: + pass #node does not exist + return nodes + +def parse_input(log_data): + nodes={} + for line in log_data: + if BEGIN_NODES in line : + nodes={} + for line in log_data: + if END_NODES in line : + break + l = line.replace('\n','').split() #TODO unhack me + nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] } + if BEGIN_SUBNET in line : + for line in log_data: + if END_SUBNET in line : + break + l = line.replace('\n','').split() + if not nodes[l[2]].get('internal-ip',False): + nodes[l[2]]['internal-ip'] = [] + nodes[l[2]]['internal-ip'].append(l[0].split('#')[0]) + if BEGIN_EDGES in line : + edges = {} + for line in log_data: + if END_EDGES in line : + break + l = line.replace('\n','').split() + if not nodes[l[0]].has_key('to') : + nodes[l[0]]['to'] = [] + nodes[l[0]]['to'].append( + {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] }) + return nodes + + +if __name__ == '__main__': + import subprocess,time + if is_legacy: + subprocess.call(["pkill","-SIGUSR2", "tincd"]) + time.sleep(1) + print json.dumps(parse_input((get_tinc_block(SYSLOG_FILE)))) + else: + print json.dumps(parse_new_input()) + diff --git a/retiolum/scripts/github_listener/github_listener.conf b/retiolum/scripts/github_listener/github_listener.conf index 88c8ea60..c4f0a8b6 100644 --- a/retiolum/scripts/github_listener/github_listener.conf +++ b/retiolum/scripts/github_listener/github_listener.conf @@ -1,4 +1,5 @@ [program:github_listener] -command=/opt/github_listener/retiolum/scripts/github_listener/listen 5432 +command=nc -lvv -p 5432 -c "./handle_request /opt/github_listener/retiolum/hosts /var/www/euer.krebsco.de/retiolum/" user=tinc -environment=HOSTFOLDER='/opt/github_listener/retiolum/hosts',WEBDIR='/srv/http/pub/retiolum/' +directory=/krebs/retiolum/scripts/github_listener/ +autorestart=true diff --git a/retiolum/scripts/github_listener/handle_request b/retiolum/scripts/github_listener/handle_request index 2df05ae0..5b42524a 100755 --- a/retiolum/scripts/github_listener/handle_request +++ b/retiolum/scripts/github_listener/handle_request @@ -3,14 +3,21 @@ # WEBDIR # HOSTFOLDER set -euf -HOSTFOLDER=${HOSTFOLDER:-../../hosts} +if [ "x${2:-}" = x ];then + echo "usage: $0 HOSTDIRECTORY WEBDIRECTORY" + exit 1 +fi +export HOSTDIR=${1:-../../hosts} +WEBDIR=${2:-/var/www/euer.krebsco.de/retiolum/} +echo "sorry for keeping you waiting, please be patient" + cd $(dirname $(readlink -f $0)) -cd "$HOSTFOLDER" + +cd "$HOSTDIR" git pull origin master >&2 +echo "First step done" cd - >&2 - -echo "sorry for keeping you waiting, please be patient" -../../bin/create-supernode-tar +../../bin/create-supernode-tar $WEBDIR echo "almost done..." -../../bin/create-host-tar +../../bin/create-host-tar $WEBDIR echo "Thank you for your patience!" diff --git a/retiolum/scripts/github_listener/listen b/retiolum/scripts/github_listener/listen deleted file mode 100755 index 30f0c4b5..00000000 --- a/retiolum/scripts/github_listener/listen +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -PORT=${1:-5432} -# eloop pattern -cd $(dirname $(readlink -f $0)) -while sleep 1; do - nc -lvv -p $PORT -e ./handle_request && logger "successfully built retiolum hosts tarball" - -done |