summaryrefslogtreecommitdiffstats
path: root/retiolum
diff options
context:
space:
mode:
Diffstat (limited to 'retiolum')
-rw-r--r--retiolum/bin/nodes-to-json (renamed from retiolum/bin/nodes-to-json.py)0
l---------retiolum/bin/tinc2json1
l---------retiolum/bin/tinc_stats2json1
-rw-r--r--retiolum/scripts/adv_graphgen/DEPS2
-rwxr-xr-xretiolum/scripts/adv_graphgen/all_the_graphs.sh2
-rw-r--r--retiolum/scripts/adv_graphgen/map.html88
-rwxr-xr-xretiolum/scripts/adv_graphgen/parse_tinc_anon.py93
-rwxr-xr-xretiolum/scripts/adv_graphgen/parse_tinc_stats.py157
-rwxr-xr-xretiolum/scripts/adv_graphgen/tinc_stats/Geo.py57
-rw-r--r--retiolum/scripts/adv_graphgen/tinc_stats/Graph.py12
-rw-r--r--retiolum/scripts/adv_graphgen/tinc_stats/__init__.py1
11 files changed, 155 insertions, 259 deletions
diff --git a/retiolum/bin/nodes-to-json.py b/retiolum/bin/nodes-to-json
index ca9d3c8c..ca9d3c8c 100644
--- a/retiolum/bin/nodes-to-json.py
+++ b/retiolum/bin/nodes-to-json
diff --git a/retiolum/bin/tinc2json b/retiolum/bin/tinc2json
new file mode 120000
index 00000000..31bd0775
--- /dev/null
+++ b/retiolum/bin/tinc2json
@@ -0,0 +1 @@
+../scripts/adv_graphgen/tinc_stats/Log2JSON.py \ No newline at end of file
diff --git a/retiolum/bin/tinc_stats2json b/retiolum/bin/tinc_stats2json
deleted file mode 120000
index 413b778c..00000000
--- a/retiolum/bin/tinc_stats2json
+++ /dev/null
@@ -1 +0,0 @@
-../scripts/adv_graphgen/tinc_stats2json \ No newline at end of file
diff --git a/retiolum/scripts/adv_graphgen/DEPS b/retiolum/scripts/adv_graphgen/DEPS
index f14c0aa6..b9a591d3 100644
--- a/retiolum/scripts/adv_graphgen/DEPS
+++ b/retiolum/scripts/adv_graphgen/DEPS
@@ -1,2 +1,4 @@
graphviz
imagemagick
+python2
+pygeoip
diff --git a/retiolum/scripts/adv_graphgen/all_the_graphs.sh b/retiolum/scripts/adv_graphgen/all_the_graphs.sh
index 2811ebb7..381c7aeb 100755
--- a/retiolum/scripts/adv_graphgen/all_the_graphs.sh
+++ b/retiolum/scripts/adv_graphgen/all_the_graphs.sh
@@ -10,6 +10,8 @@
INTERNAL_FOLDER=/var/www/euer/graphs/retiolum
begin=`timer`
export GRAPHITE_HOST="no-omo"
+ export GEOCTIYDB="$PWD/GeoLiteCity.dat"
+ (python tinc_stats/Log2JSON.py | python tinc_stats/Geo.py > $INTERNAL_FOLDER/marker.json)&
(./anonytize.sh $EXTERNAL_FOLDER && echo "`date` anonytize done" >> /tmp/build_graph)&
(./sanitize.sh $INTERNAL_FOLDER && echo "`date` sanitize done" >> /tmp/build_graph)&
# wait
diff --git a/retiolum/scripts/adv_graphgen/map.html b/retiolum/scripts/adv_graphgen/map.html
new file mode 100644
index 00000000..ef8a0565
--- /dev/null
+++ b/retiolum/scripts/adv_graphgen/map.html
@@ -0,0 +1,88 @@
+
+
+<!DOCTYPE html>
+<html>
+ <head>
+ <title>Simple Map</title>
+ <meta name="viewport" content="initial-scale=1.0, user-scalable=no">
+ <meta charset="utf-8">
+ <style>
+ html, body, #map-canvas {
+ margin: 0;
+ padding: 0;
+ height: 100%;
+ }
+ </style>
+ <script src="https://maps.googleapis.com/maps/api/js?v=3.exp&sensor=false"></script>
+ <script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.7/jquery.min.js"></script>
+ <script type="text/javascript" src="jquery.ui.map.js"></script>
+ <script>
+ var map;
+ var markerlist ={};
+ function addInfoWindow(marker, message) {
+ var info = message;
+
+ var infoWindow = new google.maps.InfoWindow({
+ content: message
+ });
+
+ google.maps.event.addListener(marker, 'click', function () {
+
+ // close all the infowindows
+ $.each(markerlist,function (k,v){
+ v["infowin"].close()
+ });
+ infoWindow.open(map, marker);
+ });
+ return infoWindow;
+ }
+ function initialize() {
+ var mapOptions = {
+ zoom: 5,
+ center: new google.maps.LatLng(51,9),
+ mapTypeId: google.maps.MapTypeId.ROADMAP
+ };
+ map = new google.maps.Map(document.getElementById('map-canvas'),
+ mapOptions);
+
+ $.getJSON("marker.json",function (data){
+ $.each(data,function (k,v) {
+
+ // add initial marker
+ var pt=new google.maps.LatLng(v["latitude"],v["longitude"])
+ var marker= new google.maps.Marker({
+ id: k,map:map,title: k,position:pt,content:k})
+
+ // add edges
+ $.each(v["to"],function(iter,val){
+ var line = [
+ pt,
+ new google.maps.LatLng(val['latitude'],val['longitude'])
+ ]
+
+ var connector = new google.maps.Polyline({
+ path:line,
+ strokeColor: "#FF0000",
+ strokeOpacity: 1.0,
+ strokeWeight: 2
+ })
+ connector.setMap(map);
+ val["line"]=connector
+
+ })
+
+
+ marker["infowin"]=addInfoWindow(marker,k+'<br/>'+v["city"]);
+ markerlist[k] = marker;
+ });
+ })
+ };
+
+ google.maps.event.addDomListener(window, 'load', initialize);
+
+ </script>
+ </head>
+ <body>
+ <div id="map-canvas"></div>
+ </body>
+</html>
diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_anon.py b/retiolum/scripts/adv_graphgen/parse_tinc_anon.py
deleted file mode 100755
index 05b82828..00000000
--- a/retiolum/scripts/adv_graphgen/parse_tinc_anon.py
+++ /dev/null
@@ -1,93 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf8 -*-
-from tinc_stats.Graphite import GraphiteSender
-from time import time
-
-try:
- import os
- gr = GraphiteSender(os.environ.get("GRAPHITE_HOST","localhost"))
- begin = time()
-except Exception as e:
- sys.stderr.write( "Cannot connect to graphite: " + str(e))
-
-
-def write_digraph(nodes):
- """
- writes the complete digraph in dot format
- """
- print ('digraph retiolum {')
- #print (' graph [center rankdir=LR packMode="clust"]')
- print (' graph [center packMode="clust"]')
- print (' node[shape=circle,style=filled,fillcolor=grey]')
- print (' overlap=false')
- generate_stats(nodes)
- merge_edges(nodes)
- nodes = anon_nodes(nodes)
- for k,v in nodes.iteritems():
- write_node(k,v)
- write_stat_node(nodes)
- print ('}')
-
-def anon_nodes(nodes):
- #anonymizes all nodes
- i = "0"
- newnodes = {}
- for k,v in nodes.iteritems():
- for nodek,node in nodes.iteritems():
- for to in node['to']:
- if to['name'] == k:
- to['name'] = i
- newnodes[i] = v
- i = str(int(i)+1)
- return newnodes
-
-def write_stat_node(nodes):
- ''' Write a `stats` node in the corner
- This node contains infos about the current number of active nodes and connections inside the network
- '''
- from time import localtime,strftime
- num_conns = 0
- num_nodes = len(nodes)
- for k,v in nodes.iteritems():
- num_conns+= len(v['to'])
- node_text = " stats_node [shape=box,label=\"Statistics\\l"
- node_text += "Build Date : %s\\l" % strftime("%Y-%m-%d %H:%M:%S",localtime())
- node_text += "Active Nodes: %s\\l" % num_nodes
- node_text += "Connections : %s\\l" % num_conns
- node_text += "\""
- node_text += ",fillcolor=green"
- node_text += "]"
- print(node_text)
-
-
-
-def write_node(k,v):
- """ writes a single node and its edges
- edges are weightet with the informations inside the nodes provided by
- tinc
- """
-
- node = " "+k #+"[label=\""
- print node
-
- for con in v.get('to',[]):
- label = con['weight']
- w = int(con['weight'])
- weight = str(1000 - (((w - 150) * (1000 - 0)) / (1000 -150 )) + 0)
-
- length = str(float(w)/1500)
- if float(weight) < 0 :
- weight= "1"
-
- edge = " "+k+ " -> " +con['name'] + " [label="+label + " weight="+weight
- if con.get('bidirectional',False):
- edge += ",dir=both"
- edge += "]"
- print edge
-
-if __name__ == "__main__":
- nodes = delete_unused_nodes(json.load(sys.stdin))
- write_digraph(nodes)
- try:
- gr.send("graph.anon_build_time",(time()-begin)*1000)
- except Exception as e: pass
diff --git a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py b/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
deleted file mode 100755
index c23e348d..00000000
--- a/retiolum/scripts/adv_graphgen/parse_tinc_stats.py
+++ /dev/null
@@ -1,157 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf8 -*-
-from tinc_stats.BackwardsReader import BackwardsReader
-from tinc_stats.Graph import generate_stats,delete_unused_nodes,merge_edges,get_node_avg_weight
-from tinc_stats.Supernodes import check_all_the_super
-from tinc_stats.Availability import get_node_availability
-import sys,json
-from tinc_stats.Graphite import GraphiteSender
-from time import time
-
-try:
- import os
- gr = GraphiteSender(os.environ.get("GRAPHITE_HOST","localhost"))
- begin = time()
-except Exception as e:
- sys.stderr.write( "Cannot connect to graphite: " + str(e))
-
-supernodes= [ ]
-
-DUMP_FILE = "/krebs/db/availability"
-
-def write_digraph(nodes):
- """
- writes the complete digraph in dot format
- """
- print ('digraph retiolum {')
- #print (' graph [center rankdir=LR packMode="clust"]')
- print (' graph [center packMode="clust"]')
- print (' node[shape=box,style=filled,fillcolor=grey]')
- print (' overlap=false')
- generate_stats(nodes)
- merge_edges(nodes)
- write_stat_node(nodes)
- for k,v in nodes.iteritems():
- write_node(k,v)
- print ('}')
-
-def dump_graph(nodes):
- from time import time
- graph = {}
- graph['nodes'] = nodes
- graph['timestamp'] = time()
- f = open(DUMP_FILE,'a')
- json.dump(graph,f)
- f.write('\n')
- f.close()
-
-def write_stat_node(nodes):
- ''' Write a `stats` node in the corner
- This node contains infos about the current number of active nodes and connections inside the network
- '''
- from time import localtime,strftime
- num_conns = 0
- num_nodes = len(nodes)
- try:
- msg = '%s.num_nodes %d %d\r\n' %(g_path,num_nodes,begin)
- s.send(msg)
- except Exception as e: pass
- #except: pass
- for k,v in nodes.iteritems():
- num_conns+= len(v['to'])
- node_text = " stats_node [label=\"Statistics\\l"
- node_text += "Build Date : %s\\l" % strftime("%Y-%m-%d %H:%M:%S",localtime())
- node_text += "Active Nodes: %s\\l" % num_nodes
- node_text += "Connections : %s\\l" % num_conns
- node_text += "\""
- node_text += ",fillcolor=green"
- node_text += "]"
- print(node_text)
-
-def generate_stats(nodes):
- """ Generates some statistics of the network and nodes
- """
- jlines = []
- try:
- f = BackwardsReader(DUMP_FILE)
- lines_to_use = 1000
- while True:
- if lines_to_use == 0: break
- line = f.readline()
- if not line: break
- jline = json.loads(line)
- if not jline['nodes']: continue
-
- jlines.append(jline)
- lines_to_use -=1
- except Exception,e: sys.stderr.write(str(e))
- for k,v in nodes.iteritems():
- conns = v.get('to',[])
- for c in conns: #sanitize weights
- if float(c['weight']) > 9000: c['weight'] = str(9001)
- elif float(c['weight']) < 0: c['weight'] = str(0)
- v['num_conns'] = len(conns)
- v['avg_weight'] = get_node_avg_weight(conns)
- v['availability'] = get_node_availability(k,jlines)
- sys.stderr.write( "%s -> %f\n" %(k ,v['availability']))
-
-
-
-def write_node(k,v):
- """ writes a single node and its edges
- edges are weightet with the informations inside the nodes provided by
- tinc
- """
-
- node = " "+k+"[label=\""
- node += k+"\\l"
- node += "availability: %f\\l" % v['availability']
- #node += "avg weight: %.2f\\l" % v['avg_weight']
- if v.has_key('num_conns'):
- node += "Num Connects:"+str(v['num_conns'])+"\\l"
- node += "external:"+v['external-ip']+":"+v['external-port']+"\\l"
- for addr in v.get('internal-ip',['dunno lol']): #['¯\\\\(°_o)/¯']):
- node += "internal:%s\\l"%addr
- node +="\""
-
- # warning if node only has one connection
- if v['num_conns'] == 1:
- node += ",fillcolor=red"
- elif k in supernodes:
- node += ",fillcolor=steelblue1"
- #node +=",group=\""+v['external-ip'].replace(".","")+"\""
- node += "]"
- print node
-
- for con in v.get('to',[]):
- label = con['weight']
- w = int(con['weight'])
- weight = str(1000 - (((w - 150) * (1000 - 0)) / (1000 -150 )) + 0)
-
- length = str(float(w)/1500)
- #weight = "1000" #str(300/float(con['weight']))
- #weight = str((100/float(con['weight'])))
- #weight = str(-1 * (200-100000/int(con['weight'])))
- if float(weight) < 0 :
- weight= "1"
-
- #sys.stderr.write(weight + ":"+ length +" %s -> " %k + str(con) + "\n")
- edge = " "+k+ " -> " +con['name'] + "[label="+label + " weight="+weight #+ " minlen="+length
- if con.get('bidirectional',False):
- edge += ",dir=both"
- edge += "]"
- print edge
-
-if __name__ == "__main__":
- nodes = json.load(sys.stdin)
- nodes = delete_unused_nodes(nodes)
- for supernode,addr in check_all_the_super():
- supernodes.append(supernode)
- try:
- dump_graph(nodes)
- except Exception,e:
- sys.stderr.write("Cannot dump graph: %s" % str(e))
- write_digraph(nodes)
- try:
- gr.send("graph.anon_build_time",(time()-begin)*1000)
- except Exception as e: pass
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py b/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py
index ba3f6738..038ca9c0 100755
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/Geo.py
@@ -1,13 +1,54 @@
#!/usr/bin/python3
# -*- coding: utf8 -*-
-import sys,json
-from Graph import delete_unused_nodes
+import sys,json,os
+from Graph import delete_unused_nodes,resolve_myself
+GEODB=os.environ.get("GEOCITYDB","GeoLiteCity.dat")
-if __name__ == "__main__":
+def add_geo(nodes):
+ from pygeoip import GeoIP
+ gi = GeoIP(GEODB)
+
+ for k,v in nodes.iteritems():
+ try:
+ nodes[k].update(gi.record_by_addr(v["external-ip"]))
+ except Exception as e:
+ sys.stderr.write(str(e))
+ sys.stderr.write("Cannot determine GeoData for %s\n"%k)
+
+ return nodes
+def add_coords_to_edges(nodes):
from pygeoip import GeoIP
- gi = GeoIP("GeoLiteCity.dat")
- for node,data in delete_unused_nodes(json.load(sys.stdin)).items():
+ gi = GeoIP(GEODB)
+
+ for k,v in nodes.iteritems():
+ for i,j in enumerate(v.get("to",[])):
+ data=gi.record_by_addr(j["addr"])
+ try:
+ j["latitude"]=data["latitude"]
+ j["longitude"]=data["longitude"]
+ except Exception as e: pass
+
+ return nodes
+
+def add_jitter(nodes):
+ from random import random
+ #add a bit of jitter to all of the coordinates
+ max_jitter=0.005
+ for k,v in nodes.iteritems():
+ jitter_lat= max_jitter -random()*max_jitter*2
+ jitter_long= max_jitter -random()*max_jitter*2
try:
- print ("%s in %s"%(node,gi.record_by_addr(data["external-ip"])["city"]))
- except:
- print ("%s myself"%node)
+ v["latitude"]= v["latitude"] + jitter_lat
+ v["longitude"]= v["longitude"] + jitter_long
+ for nodek,node in nodes.iteritems():
+ for to in node['to']:
+ if to['name'] == k:
+ to['latitude'] = v["latitude"]
+ to['longitude'] = v["longitude"]
+ except Exception as e: pass
+ return nodes
+
+if __name__ == "__main__":
+ import json
+ nodes = add_jitter(add_coords_to_edges(add_geo(resolve_myself(delete_unused_nodes(json.load(sys.stdin))))))
+ print (json.dumps(nodes))
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py b/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py
index da0ed828..18c3d545 100644
--- a/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/Graph.py
@@ -7,6 +7,18 @@ from Availability import get_node_availability
import sys,json
from time import time
DUMP_FILE = "/krebs/db/availability"
+
+
+def resolve_myself(nodes):
+ #resolve MYSELF to the real ip
+ for k,v in nodes.iteritems():
+ if v["external-ip"] == "MYSELF":
+ for nodek,node in nodes.iteritems():
+ for to in node['to']:
+ if to['name'] == k:
+ v["external-ip"] = to["addr"]
+ return nodes
+
def dump_graph(nodes):
from time import time
graph = {}
diff --git a/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py b/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py
index 62f541d1..d4a686bf 100644
--- a/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py
+++ b/retiolum/scripts/adv_graphgen/tinc_stats/__init__.py
@@ -4,3 +4,4 @@ import Log2JSON
import Supernodes
import Geo
import Graph
+import Graphite
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-06 04:46:45 +0000