diff options
| -rwxr-xr-x | retiolum/scripts/adv_graphgen/all_the_graphs.sh | 3 | ||||
| -rwxr-xr-x | retiolum/scripts/adv_graphgen/anonytize.sh | 2 | ||||
| -rwxr-xr-x | retiolum/scripts/adv_graphgen/parse_tinc_anon.py | 17 | ||||
| -rwxr-xr-x | retiolum/scripts/adv_graphgen/parse_tinc_stats.py | 34 | ||||
| -rwxr-xr-x | retiolum/scripts/adv_graphgen/sanitize.sh | 2 | ||||
| -rwxr-xr-x | util/bin/statsc | 4 | ||||
| -rwxr-xr-x | util/bin/timer | 14 | 
7 files changed, 54 insertions, 22 deletions
| diff --git a/retiolum/scripts/adv_graphgen/all_the_graphs.sh b/retiolum/scripts/adv_graphgen/all_the_graphs.sh index 2a41df38..92c89cc9 100755 --- a/retiolum/scripts/adv_graphgen/all_the_graphs.sh +++ b/retiolum/scripts/adv_graphgen/all_the_graphs.sh @@ -2,6 +2,7 @@  echo "`date` begin all graphs" >> /tmp/build_graph  cd $(dirname $(readlink -f $0))  PATH=$PATH:../../../util/bin/ +export LOG_FILE=/var/log/retiolum.log  begin=`timer`  (./anonytize.sh /srv/http/pub/graphs/retiolum/ && echo "`date` anonytize done" >> /tmp/build_graph)&  (./sanitize.sh /srv/http/priv/graphs/retiolum/ && echo "`date` sanitize done" >> /tmp/build_graph)& @@ -10,4 +11,4 @@ do    echo $job    wait $job || echo "$job failed!"  done -statsc "graphs.buildtime:$(timer $begin)|ms" +graphitec "retiolum.graph.buildtime" "$(timer $begin)" diff --git a/retiolum/scripts/adv_graphgen/anonytize.sh b/retiolum/scripts/adv_graphgen/anonytize.sh index 1ebfe972..d49793cb 100755 --- a/retiolum/scripts/adv_graphgen/anonytize.sh +++ b/retiolum/scripts/adv_graphgen/anonytize.sh @@ -5,7 +5,7 @@ GRAPH_SETTER1=dot  GRAPH_SETTER2=circo  GRAPH_SETTER3='neato -Goverlap=prism '  GRAPH_SETTER4=sfdp -LOG_FILE=/var/log/syslog +LOG_FILE=${LOG_FILE:-/var/log/syslog}  TYPE=svg  TYPE2=png  OPENER=/bin/true diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_anon.py b/retiolum/scripts/adv_graphgen/parse_tinc_anon.py index 3b0383da..c603d398 100755 --- a/retiolum/scripts/adv_graphgen/parse_tinc_anon.py +++ b/retiolum/scripts/adv_graphgen/parse_tinc_anon.py @@ -3,6 +3,17 @@  from BackwardsReader import BackwardsReader  import sys,json  #supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ] +try: +  import socket +  from time import time +  host = "localhost" +  port = 2003 +  g_path = "retiolum" +  begin = time() +  s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)  +  s.connect((host,port)) +except Exception as e: +  print >>sys.stderr, "Cannot connect to graphite: " + str(e)  """ TODO: Refactoring needed to pull the edges out of the node structures again,  it should be easier to handle both structures"""  DUMP_FILE = "/krebs/db/availability" @@ -133,3 +144,9 @@ def decode_input(FILE):  nodes = decode_input(sys.stdin)  nodes = delete_unused_nodes(nodes)  write_digraph(nodes) +try:  +  end = time() +  msg = '%s.graph.anon_build_time %d %d\n' % (g_path,((end-begin)*1000),end) +  s.send(msg) +  s.close() +except Exception as e: print >>sys.stderr, e diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py index c676ad20..a255a918 100755 --- a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py +++ b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py @@ -3,12 +3,16 @@  from BackwardsReader import BackwardsReader  import sys,json  try: -  import statsd -except: -  print >>sys.stderr,'!! no statsd installed, try `pip install statsd`' -  timer = statsd.Timer("graph.detail") -  timer.start() -  gauge = statsd.Gauge('graph.detail') +  from time import time +  import socket +  host = "localhost" +  port = 2003 +  g_path = "retiolum" +  begin = time() +  s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)  +  s.connect((host,port)) +except Exception as e: +  print >>sys.stderr, "Cannot connect to graphite: " + str(e)  supernodes= [ "kaah","supernode","euer","pa_sharepoint","oxberg" ]  """ TODO: Refactoring needed to pull the edges out of the node structures again, @@ -44,8 +48,12 @@ def write_stat_node(nodes):    '''    num_conns = 0    num_nodes = len(nodes) -  try: gauge.send('num_nodes',num_nodes) -  except: pass +  try:  +    msg = '%s.num_nodes %d %d\n' %(g_path,num_nodes,begin) +    s.send(msg) +    #print >>sys.stderr, msg +  except Exception as e: print sys.stderr,e +  #except: pass    for k,v in nodes.iteritems():      num_conns+= len(v['to'])    node_text = "  stats_node [label=\"Statistics\\l" @@ -193,5 +201,11 @@ try:  except Exception,e:    sys.stderr.write("Cannot dump graph: %s" % str(e))  write_digraph(nodes) -try: timer.stop("execution_time") -except: pass + +try:  +  end = time() +  msg = '%s.graph.detail_build_time %d %d\n' % (g_path,((end-begin)*1000),end) +  s.send(msg) +  print >>sys.stderr,msg +  s.close() +except Exception as e: print >>sys.stderr, e diff --git a/retiolum/scripts/adv_graphgen/sanitize.sh b/retiolum/scripts/adv_graphgen/sanitize.sh index 50f1659f..c46662f3 100755 --- a/retiolum/scripts/adv_graphgen/sanitize.sh +++ b/retiolum/scripts/adv_graphgen/sanitize.sh @@ -5,7 +5,7 @@ GRAPH_SETTER1=dot  GRAPH_SETTER2=circo  GRAPH_SETTER3='neato -Goverlap=prism '  GRAPH_SETTER4=sfdp -LOG_FILE=/var/log/syslog +LOG_FILE=${LOG_FILE:-/var/log/syslog}  TYPE=svg  TYPE2=png  OPENER=/bin/true diff --git a/util/bin/statsc b/util/bin/statsc index ce2c70b6..ff86ccb8 100755 --- a/util/bin/statsc +++ b/util/bin/statsc @@ -1,4 +1,4 @@  #!/bin/sh  # see https://github.com/etsy/statsd for syntax -[[ $# -eq 0 ]] && echo "usage: $0 MESSAGE" -echo "$1" | nc localhost 8126 +[ $# -eq 0 ] && echo "usage: $0 MESSAGE" +echo "$1" | nc -w 1 -u 127.0.0.1 8125 diff --git a/util/bin/timer b/util/bin/timer index 98c43063..0ccf514a 100755 --- a/util/bin/timer +++ b/util/bin/timer @@ -1,14 +1,14 @@  #!/bin/bash - +curr=$(date '+%s%N') +etime=${curr:0:${#curr}-6}  if [[ $# -eq 0 ]];then -  echo $(date '+%s') +  echo $etime  else -  [ "$1" -eq "-h" -o "$1" -eq "--help" ] && \ -    echo "usage: $0 [start_time]" && \ -    echo "  if no start_time is given, return the currentime" && \ -    echo "  if start_time is given, return the difference" +  [ "$1" == "-h" -o "$1" == "--help" ] && \ +    echo "usage: $0 [start_ms]" && \ +    echo "  if no start_ms is given, return the currentime" && \ +    echo "  if start_ms is given, return the difference" && exit 0    stime=$1 -  etime=$(date '+%s')    [[ -z "$stime" ]] && stime=$etime    echo $((etime - stime))  fi | 
