From 1e2f845e906e9290997365661fa6c753e3868702 Mon Sep 17 00:00:00 2001
From: EUcancER <root@euer.krebsco.de>
Date: Wed, 30 May 2012 12:45:11 +0200
Subject: fix package manager order

---
 punani/bin/punani | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/punani/bin/punani b/punani/bin/punani
index 97c7ac85..23ba5e00 100755
--- a/punani/bin/punani
+++ b/punani/bin/punani
@@ -8,26 +8,28 @@ PKGS="$*"
 ## find package manager
 if ! :; then : # dummy case, so the rest has a common format
 
-elif for PACKER_CMD in yum
+elif for PACKER_CMD in aptitude apt-get
     do type $PACKER_CMD 2>/dev/null 1>&2 && break; done; then
   INSTALL_PARAM='-y install'
   REMOVE_PARAM='-y remove'
 
-elif for PACKER_CMD in brew
-    do type $PACKER_CMD 2>/dev/null 1>&2 && break; done; then
-  INSTALL_PARAM='install'
-  REMOVE_PARAM='remove'
-
 elif for PACKER_CMD in bauerbill packer yaourt pacman
     do type $PACKER_CMD 2>/dev/null 1>&2 && break; done; then
   INSTALL_PARAM='--noconfirm -S --needed'
   REMOVE_PARAM='-Rcs'
 
-elif for PACKER_CMD in aptitude apt-get
+elif for PACKER_CMD in yum
     do type $PACKER_CMD 2>/dev/null 1>&2 && break; done; then
   INSTALL_PARAM='-y install'
   REMOVE_PARAM='-y remove'
 
+elif for PACKER_CMD in brew
+    do type $PACKER_CMD 2>/dev/null 1>&2 && break; done; then
+  INSTALL_PARAM='install'
+  REMOVE_PARAM='remove'
+
+
+
 else
   echo "Error 2: no known package manager found; no punani for you!" >&2
   exit 23
-- 
cgit v1.2.3


From 6deac5efc1c0207d71006a7941c74f66a0bc0a2f Mon Sep 17 00:00:00 2001
From: EUcancER <root@euer.krebsco.de>
Date: Fri, 1 Jun 2012 14:51:31 +0200
Subject: add timer util

---
 util/bin/timer | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100755 util/bin/timer

diff --git a/util/bin/timer b/util/bin/timer
new file mode 100755
index 00000000..98c43063
--- /dev/null
+++ b/util/bin/timer
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+if [[ $# -eq 0 ]];then
+  echo $(date '+%s')
+else
+  [ "$1" -eq "-h" -o "$1" -eq "--help" ] && \
+    echo "usage: $0 [start_time]" && \
+    echo "  if no start_time is given, return the currentime" && \
+    echo "  if start_time is given, return the difference"
+  stime=$1
+  etime=$(date '+%s')
+  [[ -z "$stime" ]] && stime=$etime
+  echo $((etime - stime))
+fi
-- 
cgit v1.2.3


From d72588e1da8c075daf791256593be20c9b67ea25 Mon Sep 17 00:00:00 2001
From: EUcancER <root@euer.krebsco.de>
Date: Fri, 1 Jun 2012 14:53:47 +0200
Subject: //util/bin add statsc

---
 util/bin/statsc | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100755 util/bin/statsc

diff --git a/util/bin/statsc b/util/bin/statsc
new file mode 100755
index 00000000..ce2c70b6
--- /dev/null
+++ b/util/bin/statsc
@@ -0,0 +1,4 @@
+#!/bin/sh
+# see https://github.com/etsy/statsd for syntax
+[[ $# -eq 0 ]] && echo "usage: $0 MESSAGE"
+echo "$1" | nc localhost 8126
-- 
cgit v1.2.3


From ff93336964b5c59bb2e7ddfb790884593d8276ba Mon Sep 17 00:00:00 2001
From: EUcancER <root@euer.krebsco.de>
Date: Fri, 1 Jun 2012 15:31:44 +0200
Subject: //retiolum: adv_graphgen send stats via statsd

---
 retiolum/scripts/adv_graphgen/all_the_graphs.sh   | 10 +++++++++-
 retiolum/scripts/adv_graphgen/parse_tinc_stats.py | 12 ++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/retiolum/scripts/adv_graphgen/all_the_graphs.sh b/retiolum/scripts/adv_graphgen/all_the_graphs.sh
index 36b37f03..2a41df38 100755
--- a/retiolum/scripts/adv_graphgen/all_the_graphs.sh
+++ b/retiolum/scripts/adv_graphgen/all_the_graphs.sh
@@ -1,5 +1,13 @@
-#!/bin/sh
+#!/bin/bash
 echo "`date` begin all graphs" >> /tmp/build_graph
 cd $(dirname $(readlink -f $0))
+PATH=$PATH:../../../util/bin/
+begin=`timer`
 (./anonytize.sh /srv/http/pub/graphs/retiolum/ && echo "`date` anonytize done" >> /tmp/build_graph)&
 (./sanitize.sh /srv/http/priv/graphs/retiolum/ && echo "`date` sanitize done" >> /tmp/build_graph)&
+for job in `jobs -p`
+do
+  echo $job
+  wait $job || echo "$job failed!"
+done
+statsc "graphs.buildtime:$(timer $begin)|ms"
diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
index 54dcc4ab..c676ad20 100755
--- a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
+++ b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
@@ -2,6 +2,14 @@
 # -*- coding: utf8 -*-
 from BackwardsReader import BackwardsReader
 import sys,json
+try:
+  import statsd
+except:
+  print >>sys.stderr,'!! no statsd installed, try `pip install statsd`'
+  timer = statsd.Timer("graph.detail")
+  timer.start()
+  gauge = statsd.Gauge('graph.detail')
+
 supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ]
 """ TODO: Refactoring needed to pull the edges out of the node structures again,
 it should be easier to handle both structures"""
@@ -36,6 +44,8 @@ def write_stat_node(nodes):
   '''
   num_conns = 0
   num_nodes = len(nodes)
+  try: gauge.send('num_nodes',num_nodes)
+  except: pass
   for k,v in nodes.iteritems():
     num_conns+= len(v['to'])
   node_text = "  stats_node [label=\"Statistics\\l"
@@ -183,3 +193,5 @@ try:
 except Exception,e:
   sys.stderr.write("Cannot dump graph: %s" % str(e))
 write_digraph(nodes)
+try: timer.stop("execution_time")
+except: pass
-- 
cgit v1.2.3


From d2f737473e3bbcb7592a6745820bccd333128459 Mon Sep 17 00:00:00 2001
From: EUcancER <root@euer.krebsco.de>
Date: Sat, 2 Jun 2012 01:14:05 +0200
Subject: //retiolum adv_graphgen: now writing stats to graphite

painload now provides convenient tools to enable graphite stats generation

currently logged are:
- overall build time of graphs
- anon and detailed build times
- current number of users in the darknet
---
 retiolum/scripts/adv_graphgen/all_the_graphs.sh   |  3 +-
 retiolum/scripts/adv_graphgen/anonytize.sh        |  2 +-
 retiolum/scripts/adv_graphgen/parse_tinc_anon.py  | 17 ++++++++++++
 retiolum/scripts/adv_graphgen/parse_tinc_stats.py | 34 ++++++++++++++++-------
 retiolum/scripts/adv_graphgen/sanitize.sh         |  2 +-
 util/bin/statsc                                   |  4 +--
 util/bin/timer                                    | 14 +++++-----
 7 files changed, 54 insertions(+), 22 deletions(-)

diff --git a/retiolum/scripts/adv_graphgen/all_the_graphs.sh b/retiolum/scripts/adv_graphgen/all_the_graphs.sh
index 2a41df38..92c89cc9 100755
--- a/retiolum/scripts/adv_graphgen/all_the_graphs.sh
+++ b/retiolum/scripts/adv_graphgen/all_the_graphs.sh
@@ -2,6 +2,7 @@
 echo "`date` begin all graphs" >> /tmp/build_graph
 cd $(dirname $(readlink -f $0))
 PATH=$PATH:../../../util/bin/
+export LOG_FILE=/var/log/retiolum.log
 begin=`timer`
 (./anonytize.sh /srv/http/pub/graphs/retiolum/ && echo "`date` anonytize done" >> /tmp/build_graph)&
 (./sanitize.sh /srv/http/priv/graphs/retiolum/ && echo "`date` sanitize done" >> /tmp/build_graph)&
@@ -10,4 +11,4 @@ do
   echo $job
   wait $job || echo "$job failed!"
 done
-statsc "graphs.buildtime:$(timer $begin)|ms"
+graphitec "retiolum.graph.buildtime" "$(timer $begin)"
diff --git a/retiolum/scripts/adv_graphgen/anonytize.sh b/retiolum/scripts/adv_graphgen/anonytize.sh
index 1ebfe972..d49793cb 100755
--- a/retiolum/scripts/adv_graphgen/anonytize.sh
+++ b/retiolum/scripts/adv_graphgen/anonytize.sh
@@ -5,7 +5,7 @@ GRAPH_SETTER1=dot
 GRAPH_SETTER2=circo
 GRAPH_SETTER3='neato -Goverlap=prism '
 GRAPH_SETTER4=sfdp
-LOG_FILE=/var/log/syslog
+LOG_FILE=${LOG_FILE:-/var/log/syslog}
 TYPE=svg
 TYPE2=png
 OPENER=/bin/true
diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_anon.py b/retiolum/scripts/adv_graphgen/parse_tinc_anon.py
index 3b0383da..c603d398 100755
--- a/retiolum/scripts/adv_graphgen/parse_tinc_anon.py
+++ b/retiolum/scripts/adv_graphgen/parse_tinc_anon.py
@@ -3,6 +3,17 @@
 from BackwardsReader import BackwardsReader
 import sys,json
 #supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ]
+try:
+  import socket
+  from time import time
+  host = "localhost"
+  port = 2003
+  g_path = "retiolum"
+  begin = time()
+  s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 
+  s.connect((host,port))
+except Exception as e:
+  print >>sys.stderr, "Cannot connect to graphite: " + str(e)
 """ TODO: Refactoring needed to pull the edges out of the node structures again,
 it should be easier to handle both structures"""
 DUMP_FILE = "/krebs/db/availability"
@@ -133,3 +144,9 @@ def decode_input(FILE):
 nodes = decode_input(sys.stdin)
 nodes = delete_unused_nodes(nodes)
 write_digraph(nodes)
+try: 
+  end = time()
+  msg = '%s.graph.anon_build_time %d %d\n' % (g_path,((end-begin)*1000),end)
+  s.send(msg)
+  s.close()
+except Exception as e: print >>sys.stderr, e
diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
index c676ad20..a255a918 100755
--- a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
+++ b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
@@ -3,12 +3,16 @@
 from BackwardsReader import BackwardsReader
 import sys,json
 try:
-  import statsd
-except:
-  print >>sys.stderr,'!! no statsd installed, try `pip install statsd`'
-  timer = statsd.Timer("graph.detail")
-  timer.start()
-  gauge = statsd.Gauge('graph.detail')
+  from time import time
+  import socket
+  host = "localhost"
+  port = 2003
+  g_path = "retiolum"
+  begin = time()
+  s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 
+  s.connect((host,port))
+except Exception as e:
+  print >>sys.stderr, "Cannot connect to graphite: " + str(e)
 
 supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ]
 """ TODO: Refactoring needed to pull the edges out of the node structures again,
@@ -44,8 +48,12 @@ def write_stat_node(nodes):
   '''
   num_conns = 0
   num_nodes = len(nodes)
-  try: gauge.send('num_nodes',num_nodes)
-  except: pass
+  try: 
+    msg = '%s.num_nodes %d %d\n' %(g_path,num_nodes,begin)
+    s.send(msg)
+    #print >>sys.stderr, msg
+  except Exception as e: print sys.stderr,e
+  #except: pass
   for k,v in nodes.iteritems():
     num_conns+= len(v['to'])
   node_text = "  stats_node [label=\"Statistics\\l"
@@ -193,5 +201,11 @@ try:
 except Exception,e:
   sys.stderr.write("Cannot dump graph: %s" % str(e))
 write_digraph(nodes)
-try: timer.stop("execution_time")
-except: pass
+
+try: 
+  end = time()
+  msg = '%s.graph.detail_build_time %d %d\n' % (g_path,((end-begin)*1000),end)
+  s.send(msg)
+  print >>sys.stderr,msg
+  s.close()
+except Exception as e: print >>sys.stderr, e
diff --git a/retiolum/scripts/adv_graphgen/sanitize.sh b/retiolum/scripts/adv_graphgen/sanitize.sh
index 50f1659f..c46662f3 100755
--- a/retiolum/scripts/adv_graphgen/sanitize.sh
+++ b/retiolum/scripts/adv_graphgen/sanitize.sh
@@ -5,7 +5,7 @@ GRAPH_SETTER1=dot
 GRAPH_SETTER2=circo
 GRAPH_SETTER3='neato -Goverlap=prism '
 GRAPH_SETTER4=sfdp
-LOG_FILE=/var/log/syslog
+LOG_FILE=${LOG_FILE:-/var/log/syslog}
 TYPE=svg
 TYPE2=png
 OPENER=/bin/true
diff --git a/util/bin/statsc b/util/bin/statsc
index ce2c70b6..ff86ccb8 100755
--- a/util/bin/statsc
+++ b/util/bin/statsc
@@ -1,4 +1,4 @@
 #!/bin/sh
 # see https://github.com/etsy/statsd for syntax
-[[ $# -eq 0 ]] && echo "usage: $0 MESSAGE"
-echo "$1" | nc localhost 8126
+[ $# -eq 0 ] && echo "usage: $0 MESSAGE"
+echo "$1" | nc -w 1 -u 127.0.0.1 8125
diff --git a/util/bin/timer b/util/bin/timer
index 98c43063..0ccf514a 100755
--- a/util/bin/timer
+++ b/util/bin/timer
@@ -1,14 +1,14 @@
 #!/bin/bash
-
+curr=$(date '+%s%N')
+etime=${curr:0:${#curr}-6}
 if [[ $# -eq 0 ]];then
-  echo $(date '+%s')
+  echo $etime
 else
-  [ "$1" -eq "-h" -o "$1" -eq "--help" ] && \
-    echo "usage: $0 [start_time]" && \
-    echo "  if no start_time is given, return the currentime" && \
-    echo "  if start_time is given, return the difference"
+  [ "$1" == "-h" -o "$1" == "--help" ] && \
+    echo "usage: $0 [start_ms]" && \
+    echo "  if no start_ms is given, return the currentime" && \
+    echo "  if start_ms is given, return the difference" && exit 0
   stime=$1
-  etime=$(date '+%s')
   [[ -z "$stime" ]] && stime=$etime
   echo $((etime - stime))
 fi
-- 
cgit v1.2.3


From c04b0047db2249317a905f7d95d2fcaef6a479cc Mon Sep 17 00:00:00 2001
From: EUcancER <root@euer.krebsco.de>
Date: Tue, 5 Jun 2012 12:05:13 +0200
Subject: //retiolum all_the_graphs: update to be non-intrusive

---
 retiolum/scripts/adv_graphgen/all_the_graphs.sh | 27 +++++++++++++------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/retiolum/scripts/adv_graphgen/all_the_graphs.sh b/retiolum/scripts/adv_graphgen/all_the_graphs.sh
index 92c89cc9..0499e9c7 100755
--- a/retiolum/scripts/adv_graphgen/all_the_graphs.sh
+++ b/retiolum/scripts/adv_graphgen/all_the_graphs.sh
@@ -1,14 +1,15 @@
 #!/bin/bash
-echo "`date` begin all graphs" >> /tmp/build_graph
-cd $(dirname $(readlink -f $0))
-PATH=$PATH:../../../util/bin/
-export LOG_FILE=/var/log/retiolum.log
-begin=`timer`
-(./anonytize.sh /srv/http/pub/graphs/retiolum/ && echo "`date` anonytize done" >> /tmp/build_graph)&
-(./sanitize.sh /srv/http/priv/graphs/retiolum/ && echo "`date` sanitize done" >> /tmp/build_graph)&
-for job in `jobs -p`
-do
-  echo $job
-  wait $job || echo "$job failed!"
-done
-graphitec "retiolum.graph.buildtime" "$(timer $begin)"
+set -x
+
+(
+  1>>/tmp/build_graph
+  echo "`date` begin all graphs" >> /tmp/build_graph
+  cd $(dirname $(readlink -f $0))
+  PATH=$PATH:../../../util/bin/
+  export LOG_FILE=/var/log/retiolum.log
+  begin=`timer`
+  (./anonytize.sh /srv/http/pub/graphs/retiolum/ && echo "`date` anonytize done" >> /tmp/build_graph)&
+  (./sanitize.sh /srv/http/priv/graphs/retiolum/ && echo "`date` sanitize done" >> /tmp/build_graph)&
+  wait
+  graphitec "retiolum.graph.buildtime" "$(timer $begin)"  &>>/tmp/build_graph
+)&
-- 
cgit v1.2.3


From 6046373ce92a9c9856d828b37cda28af437e36be Mon Sep 17 00:00:00 2001
From: EUcancER <root@euer.krebsco.de>
Date: Wed, 6 Jun 2012 00:11:02 +0200
Subject: //retiolum: trying to fix all_the_graphs

---
 retiolum/scripts/adv_graphgen/all_the_graphs.sh | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/retiolum/scripts/adv_graphgen/all_the_graphs.sh b/retiolum/scripts/adv_graphgen/all_the_graphs.sh
index 0499e9c7..1640a446 100755
--- a/retiolum/scripts/adv_graphgen/all_the_graphs.sh
+++ b/retiolum/scripts/adv_graphgen/all_the_graphs.sh
@@ -1,8 +1,6 @@
 #!/bin/bash
-set -x
 
 (
-  1>>/tmp/build_graph
   echo "`date` begin all graphs" >> /tmp/build_graph
   cd $(dirname $(readlink -f $0))
   PATH=$PATH:../../../util/bin/
@@ -10,6 +8,7 @@ set -x
   begin=`timer`
   (./anonytize.sh /srv/http/pub/graphs/retiolum/ && echo "`date` anonytize done" >> /tmp/build_graph)&
   (./sanitize.sh /srv/http/priv/graphs/retiolum/ && echo "`date` sanitize done" >> /tmp/build_graph)&
-  wait
-  graphitec "retiolum.graph.buildtime" "$(timer $begin)"  &>>/tmp/build_graph
+#  wait
+#  graphitec "retiolum.graph.buildtime" "$(timer $begin)"   >> /tmp/build_graph
+  echo "`date` end all graphs" >> /tmp/build_graph
 )&
-- 
cgit v1.2.3