summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authormakefu <github@syntax-fehler.de>2013-01-04 04:31:38 +0700
committermakefu <github@syntax-fehler.de>2013-01-04 04:31:38 +0700
commit577c64f0c8e70081a2226522cbfc18c393dcb5e3 (patch)
tree231d180b730148b60d6753f8df03e17c1b871d9b
parent1027cbc903b6e1852247225ca0aa6b958cf2550c (diff)
parentc971ce27a09d4aa5e740fda3664c8a8181de3233 (diff)
Merge branch 'master' of github.com:krebscode/painload
-rwxr-xr-xretiolum/bin/create-host-tar16
-rwxr-xr-xretiolum/bin/create-supernode-tar13
l---------retiolum/bin/tinc_stats1
l---------[-rwxr-xr-x]retiolum/bin/tinc_stats2json124
-rwxr-xr-xretiolum/scripts/adv_graphgen/all_the_graphs.sh6
-rwxr-xr-xretiolum/scripts/adv_graphgen/tinc_stats2json124
-rw-r--r--retiolum/scripts/github_listener/github_listener.conf5
-rwxr-xr-xretiolum/scripts/github_listener/handle_request19
-rwxr-xr-xretiolum/scripts/github_listener/listen8
9 files changed, 169 insertions, 147 deletions
diff --git a/retiolum/bin/create-host-tar b/retiolum/bin/create-host-tar
index 9def7cec..48ce26d5 100755
--- a/retiolum/bin/create-host-tar
+++ b/retiolum/bin/create-host-tar
@@ -1,9 +1,15 @@
#!/bin/sh
-set -xeuf
-WEBDIR=${WEBDIR:-/srv/http/pub/retiolum}
+set -euf
+if [ x"${1:-}" = "x" ];then
+ echo "usage: $0 DIRECTORY"
+ exit 1
+fi
+
+WEBDIR=$1
+HOSTDIR=${HOSTDIR:-../hosts}
HOSTFILE=$WEBDIR/hosts.tar.gz
+
cd $(dirname $(readlink -f $0))
-git pull origin master || logger "cannot pull painload"
-#rm $HOSTFILE
-tar czf $HOSTFILE -C ../hosts .
+rm -f $HOSTFILE
+tar czf $HOSTFILE -C $HOSTDIR .
chmod a+rx $HOSTFILE
diff --git a/retiolum/bin/create-supernode-tar b/retiolum/bin/create-supernode-tar
new file mode 100755
index 00000000..322d428d
--- /dev/null
+++ b/retiolum/bin/create-supernode-tar
@@ -0,0 +1,13 @@
+#!/bin/sh
+set -euf
+if [ x"${1:-}" = "x" ];then
+ echo "usage: $0 DIRECTORY"
+ exit 1
+fi
+HOSTDIR=${HOSTDIR:-../hosts}
+WEBDIR=$1
+HOSTFILE=$WEBDIR/supernodes.tar.gz
+cd $(dirname $(readlink -f $0))
+rm -f $HOSTFILE
+tar czf $HOSTFILE -C $HOSTDIR `python ../scripts/adv_graphgen/find_super.py |cut -d\ -f1`
+chmod a+rx $HOSTFILE
diff --git a/retiolum/bin/tinc_stats b/retiolum/bin/tinc_stats
deleted file mode 120000
index 6a58af60..00000000
--- a/retiolum/bin/tinc_stats
+++ /dev/null
@@ -1 +0,0 @@
-/home/makefu/repos/krebs/retiolum/scripts/adv_graphgen/tinc_stats.py \ No newline at end of file
diff --git a/retiolum/bin/tinc_stats2json b/retiolum/bin/tinc_stats2json
index acadb306..413b778c 100755..120000
--- a/retiolum/bin/tinc_stats2json
+++ b/retiolum/bin/tinc_stats2json
@@ -1,123 +1 @@
-#!/usr/bin/python
-import subprocess
-import os
-import re
-import sys
-import json
-
-
-
-TINC_NETWORK =os.environ.get("TINC_NETWORK","retiolum")
-
-# is_legacy is the parameter which defines if the tinc config files are handled old fashioned (parse from syslog),
-# or if the new and hip tincctl should be used
-is_legacy= os.environ.get("TINC_LEGACY",False)
-SYSLOG_FILE = os.environ.get("LOG_FILE","/var/log/everything.log")
-
-
-# Tags and Delimiters
-TINC_TAG="tinc.%s" % TINC_NETWORK
-BEGIN_NODES = "Nodes:"
-END_NODES = "End of nodes."
-BEGIN_SUBNET = "Subnet list:"
-END_SUBNET = "End of subnet list"
-BEGIN_EDGES = "Edges:"
-END_EDGES = "End of edges."
-
-def get_tinc_block(log_file):
- """ returns an iterateable block from the given log file (syslog)
- This function became obsolete with the introduction of tincctl
- """
- from BackwardsReader import BackwardsReader
- tinc_block = []
- in_block = False
- bf = BackwardsReader(log_file)
- BOL = re.compile(".*tinc.%s\[[0-9]+\]: " % TINC_NETWORK)
- while True:
- line = bf.readline()
- if not line:
- raise Exception("end of file at log file? This should not happen!")
- line = BOL.sub('',line).strip()
-
- if END_SUBNET in line:
- in_block = True
-
- if not in_block:
- continue
-
- tinc_block.append(line)
-
- if BEGIN_NODES in line:
- break
- return reversed(tinc_block)
-
-def parse_new_input():
- nodes = {}
- pnodes = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","reachable","nodes"])
- for line in pnodes.split('\n'):
- if not line: continue
- l = line.split()
- nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] }
- psubnets = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","subnets"])
- for line in psubnets.split('\n'):
- if not line: continue
- l = line.split()
- try:
- if not nodes[l[2]].get('internal-ip',False):
- nodes[l[2]]['internal-ip'] = []
- nodes[l[2]]['internal-ip'].append(l[0].split('#')[0])
- except KeyError:
- pass # node does not exist (presumably)
- pedges = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","edges"])
- for line in pedges.split('\n'):
- if not line: continue
- l = line.split()
- try:
- if not nodes[l[0]].has_key('to') :
- nodes[l[0]]['to'] = []
- nodes[l[0]]['to'].append(
- {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] })
- except KeyError:
- pass #node does not exist
- return nodes
-
-def parse_input(log_data):
- nodes={}
- for line in log_data:
- if BEGIN_NODES in line :
- nodes={}
- for line in log_data:
- if END_NODES in line :
- break
- l = line.replace('\n','').split() #TODO unhack me
- nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] }
- if BEGIN_SUBNET in line :
- for line in log_data:
- if END_SUBNET in line :
- break
- l = line.replace('\n','').split()
- if not nodes[l[2]].get('internal-ip',False):
- nodes[l[2]]['internal-ip'] = []
- nodes[l[2]]['internal-ip'].append(l[0].split('#')[0])
- if BEGIN_EDGES in line :
- edges = {}
- for line in log_data:
- if END_EDGES in line :
- break
- l = line.replace('\n','').split()
- if not nodes[l[0]].has_key('to') :
- nodes[l[0]]['to'] = []
- nodes[l[0]]['to'].append(
- {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] })
- return nodes
-
-
-if __name__ == '__main__':
- import subprocess,time
- if is_legacy:
- subprocess.call(["pkill","-SIGUSR2", "tincd"])
- time.sleep(1)
- print json.dumps(parse_input((get_tinc_block(SYSLOG_FILE))))
- else:
- print json.dumps(parse_new_input())
-
+../scripts/adv_graphgen/tinc_stats2json \ No newline at end of file
diff --git a/retiolum/scripts/adv_graphgen/all_the_graphs.sh b/retiolum/scripts/adv_graphgen/all_the_graphs.sh
index 5533c722..8aa0ab21 100755
--- a/retiolum/scripts/adv_graphgen/all_the_graphs.sh
+++ b/retiolum/scripts/adv_graphgen/all_the_graphs.sh
@@ -5,10 +5,12 @@
cd $(dirname $(readlink -f $0))
PATH=$PATH:../../../util/bin/
export LOG_FILE=/var/log/retiolum.log
+ EXTERNAL_FOLDER=/var/www/euer.krebsco.de/graphs/retiolum
+ INTERNAL_FOLDER=/var/www/euer/graphs/retiolum
begin=`timer`
export GRAPHITE_HOST="no_omo"
- (./anonytize.sh /srv/http/pub/graphs/retiolum/ && echo "`date` anonytize done" >> /tmp/build_graph)&
- (./sanitize.sh /srv/http/priv/graphs/retiolum/ && echo "`date` sanitize done" >> /tmp/build_graph)&
+ (./anonytize.sh $EXTERNAL_FOLDER && echo "`date` anonytize done" >> /tmp/build_graph)&
+ (./sanitize.sh $INTERNAL_FOLDER && echo "`date` sanitize done" >> /tmp/build_graph)&
# wait
graphitec "retiolum.graph.buildtime" "$(timer $begin)" >> /tmp/build_graph
echo "`date` end all graphs" >> /tmp/build_graph
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats2json b/retiolum/scripts/adv_graphgen/tinc_stats2json
new file mode 100755
index 00000000..ede19b26
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/tinc_stats2json
@@ -0,0 +1,124 @@
+#!/usr/bin/python
+import subprocess
+import os
+import re
+import sys
+import json
+
+
+
+TINC_NETWORK =os.environ.get("TINC_NETWORK","retiolum")
+
+# is_legacy is the parameter which defines if the tinc config files are handled old fashioned (parse from syslog),
+# or if the new and hip tincctl should be used
+is_legacy= os.environ.get("TINC_LEGACY",False)
+SYSLOG_FILE = os.environ.get("LOG_FILE","/var/log/everything.log")
+
+
+# Tags and Delimiters
+TINC_TAG="tinc.%s" % TINC_NETWORK
+BEGIN_NODES = "Nodes:"
+END_NODES = "End of nodes."
+BEGIN_SUBNET = "Subnet list:"
+END_SUBNET = "End of subnet list"
+BEGIN_EDGES = "Edges:"
+END_EDGES = "End of edges."
+
+def get_tinc_block(log_file):
+ """ returns an iterateable block from the given log file (syslog)
+ This function became obsolete with the introduction of tincctl
+ """
+ from BackwardsReader import BackwardsReader
+ tinc_block = []
+ in_block = False
+ bf = BackwardsReader(log_file)
+ BOL = re.compile(".*tinc.%s\[[0-9]+\]: " % TINC_NETWORK)
+ while True:
+ line = bf.readline()
+ if not line:
+ raise Exception("end of file at log file? This should not happen!")
+ line = BOL.sub('',line).strip()
+
+ if END_SUBNET in line:
+ in_block = True
+
+ if not in_block:
+ continue
+
+ tinc_block.append(line)
+
+ if BEGIN_NODES in line:
+ break
+ return reversed(tinc_block)
+
+def parse_new_input():
+ nodes = {}
+ pnodes = subprocess.Popen(["tincctl","-n",TINC_NETWORK,"dump","reachable","nodes"], stdout=subprocess.PIPE).communicate()[0]
+ #pnodes = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","reachable","nodes"])
+ for line in pnodes.split('\n'):
+ if not line: continue
+ l = line.split()
+ nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] }
+ psubnets = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","subnets"])
+ for line in psubnets.split('\n'):
+ if not line: continue
+ l = line.split()
+ try:
+ if not nodes[l[2]].get('internal-ip',False):
+ nodes[l[2]]['internal-ip'] = []
+ nodes[l[2]]['internal-ip'].append(l[0].split('#')[0])
+ except KeyError:
+ pass # node does not exist (presumably)
+ pedges = subprocess.check_output(["tincctl","-n",TINC_NETWORK,"dump","edges"])
+ for line in pedges.split('\n'):
+ if not line: continue
+ l = line.split()
+ try:
+ if not nodes[l[0]].has_key('to') :
+ nodes[l[0]]['to'] = []
+ nodes[l[0]]['to'].append(
+ {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] })
+ except KeyError:
+ pass #node does not exist
+ return nodes
+
+def parse_input(log_data):
+ nodes={}
+ for line in log_data:
+ if BEGIN_NODES in line :
+ nodes={}
+ for line in log_data:
+ if END_NODES in line :
+ break
+ l = line.replace('\n','').split() #TODO unhack me
+ nodes[l[0]]= { 'external-ip': l[2], 'external-port' : l[4] }
+ if BEGIN_SUBNET in line :
+ for line in log_data:
+ if END_SUBNET in line :
+ break
+ l = line.replace('\n','').split()
+ if not nodes[l[2]].get('internal-ip',False):
+ nodes[l[2]]['internal-ip'] = []
+ nodes[l[2]]['internal-ip'].append(l[0].split('#')[0])
+ if BEGIN_EDGES in line :
+ edges = {}
+ for line in log_data:
+ if END_EDGES in line :
+ break
+ l = line.replace('\n','').split()
+ if not nodes[l[0]].has_key('to') :
+ nodes[l[0]]['to'] = []
+ nodes[l[0]]['to'].append(
+ {'name':l[2],'addr':l[4],'port':l[6],'weight' : l[10] })
+ return nodes
+
+
+if __name__ == '__main__':
+ import subprocess,time
+ if is_legacy:
+ subprocess.call(["pkill","-SIGUSR2", "tincd"])
+ time.sleep(1)
+ print json.dumps(parse_input((get_tinc_block(SYSLOG_FILE))))
+ else:
+ print json.dumps(parse_new_input())
+
diff --git a/retiolum/scripts/github_listener/github_listener.conf b/retiolum/scripts/github_listener/github_listener.conf
index 88c8ea60..c4f0a8b6 100644
--- a/retiolum/scripts/github_listener/github_listener.conf
+++ b/retiolum/scripts/github_listener/github_listener.conf
@@ -1,4 +1,5 @@
[program:github_listener]
-command=/opt/github_listener/retiolum/scripts/github_listener/listen 5432
+command=nc -lvv -p 5432 -c "./handle_request /opt/github_listener/retiolum/hosts /var/www/euer.krebsco.de/retiolum/"
user=tinc
-environment=HOSTFOLDER='/opt/github_listener/retiolum/hosts',WEBDIR='/srv/http/pub/retiolum/'
+directory=/krebs/retiolum/scripts/github_listener/
+autorestart=true
diff --git a/retiolum/scripts/github_listener/handle_request b/retiolum/scripts/github_listener/handle_request
index 2df05ae0..5b42524a 100755
--- a/retiolum/scripts/github_listener/handle_request
+++ b/retiolum/scripts/github_listener/handle_request
@@ -3,14 +3,21 @@
# WEBDIR
# HOSTFOLDER
set -euf
-HOSTFOLDER=${HOSTFOLDER:-../../hosts}
+if [ "x${2:-}" = x ];then
+ echo "usage: $0 HOSTDIRECTORY WEBDIRECTORY"
+ exit 1
+fi
+export HOSTDIR=${1:-../../hosts}
+WEBDIR=${2:-/var/www/euer.krebsco.de/retiolum/}
+echo "sorry for keeping you waiting, please be patient"
+
cd $(dirname $(readlink -f $0))
-cd "$HOSTFOLDER"
+
+cd "$HOSTDIR"
git pull origin master >&2
+echo "First step done"
cd - >&2
-
-echo "sorry for keeping you waiting, please be patient"
-../../bin/create-supernode-tar
+../../bin/create-supernode-tar $WEBDIR
echo "almost done..."
-../../bin/create-host-tar
+../../bin/create-host-tar $WEBDIR
echo "Thank you for your patience!"
diff --git a/retiolum/scripts/github_listener/listen b/retiolum/scripts/github_listener/listen
deleted file mode 100755
index 30f0c4b5..00000000
--- a/retiolum/scripts/github_listener/listen
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh
-PORT=${1:-5432}
-# eloop pattern
-cd $(dirname $(readlink -f $0))
-while sleep 1; do
- nc -lvv -p $PORT -e ./handle_request && logger "successfully built retiolum hosts tarball"
-
-done