working map

bgpdata/api.py

@@ -0,0 +1,19 @@
+#from tastypie.resources import ModelResource, ALL_WITH_RELATIONS
+#from tastypie import fields
+#from bgpdata.models import AS, CrawlRun
+#
+#class ASResource(ModelResource):
+#	crawl = fields.ForeignKey("bgpdata.api.CrawlResource", "crawl")
+#	class Meta:
+#		list_allowed_methods = ['get']
+#		detail_allowed_methods = ['get']
+#		filtering = {'crawl': ALL_WITH_RELATIONS}
+#
+#		queryset = AS.objects.all()
+#		resource_name = "as"
+#
+#class CrawlResource(ModelResource):
+#	class Meta:
+#		queryset = CrawlRun.objects.all()
+#		resource_name = "crawl"
+

bgpdata/migrations/0009_auto_20150326_2207.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bgpdata', '0008_auto_20150322_1906'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='as',
+            name='directlyCrawled',
+            field=models.BooleanField(default=False),
+            preserve_default=True,
+        ),
+        migrations.AlterField(
+            model_name='crawllog',
+            name='host',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, blank=True, to='bgpdata.ConfigHost', null=True),
+            preserve_default=True,
+        ),
+    ]

bgpdata/migrations/0010_auto_20150327_0003.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bgpdata', '0009_auto_20150326_2207'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='as',
+            name='lastSeen',
+            field=models.ForeignKey(related_name='as_lastseen', default=None, blank=True, to='bgpdata.CrawlRun', null=True),
+            preserve_default=True,
+        ),
+        migrations.AddField(
+            model_name='as',
+            name='online',
+            field=models.BooleanField(default=True),
+            preserve_default=True,
+        ),
+    ]

bgpdata/models.py

@@ -39,7 +39,7 @@ class CrawlLog(models.Model):
 	)
 
 	crawl = models.ForeignKey(CrawlRun)
-	host = models.ForeignKey(ConfigHost, null=True, blank=True)
+	host = models.ForeignKey(ConfigHost, null=True, blank=True, on_delete=models.SET_NULL)
 	logtime = models.DateTimeField(auto_now_add=True)
 	severity = models.CharField(max_length=10, choices=SEVERITY)
 	message = models.TextField()
@@ -65,6 +65,10 @@ class AS(models.Model):
 	crawl = models.ForeignKey(CrawlRun)
 	number = models.IntegerField()
 
+	directlyCrawled = models.BooleanField(default=False)
+	online = models.BooleanField(default=True)
+	lastSeen = models.ForeignKey(CrawlRun, blank=True, null=True, default=None, related_name='as_lastseen')
+
 	def __unicode__(self):
 		return u"AS %s (crawl %d)" % (self.number, self.crawl.pk)
 

bgpdata/templates/bgpdata/map-2.html

@@ -0,0 +1,114 @@
+{% extends "base.html" %}
+{% block head %}
+{% load static from staticfiles %}
+<script src="{% static "js/d3.js" %}" charset="utf-8"></script>
+<style>
+.node {
+  stroke: #fff;
+  stroke-width: 1.5px;
+}
+
+.link {
+  stroke: #999;
+  stroke-opacity: .6;
+}
+</style>
+{% endblock %}
+{% block body %}
+
+<h3>Crawl run {{crawl.pk}} from {{crawl.startTime}}</h3>
+
+<div id="plotwin"></div>
+<script type="text/javascript" charset="utf-8">
+
+var asdata = [
+{% for AS in ASses %}
+	{id: {{AS.pk}}, numer: {{AS.number}}, label: 'MAUNZ'}{%if not forloop.last%},{%endif%}
+{%endfor%}
+];
+
+var asdict = {
+{% for AS in ASses %}
+	{{AS.number}}: {{forloop.counter0}}{%if not forloop.last%},{%endif%}
+{% endfor %}
+};
+
+var peerings = [
+{% for p in peerings %}
+	{
+		source: asdict[{{p.as1.number}}],
+		target: asdict[{{p.as2.number}}],
+
+		id: {{p.pk}},
+		as1id: {{p.as1.id}},
+		as1number: {{p.as1.number}},
+		as2id: {{p.as2.id}},
+		as2number: {{p.as2.number}},
+
+		origin: "{{p.get_origin_display}}"
+	}{%if not forloop.last%},{%endif%}
+{%endfor%}
+];
+
+
+var width = 960,
+    height = 600;
+
+var svg = d3.select('#plotwin')
+			.append('svg')
+			.attr('width', width)
+			.attr('height', height);
+
+console.log(asdata);
+var force = d3.layout.force()
+	.nodes(asdata)
+	.links(peerings)
+    .charge(-200)
+    .linkDistance(70)
+    .size([width, height])
+	.start();
+   // .charge(-120)
+   // .linkDistance(30)
+
+
+var link = svg.selectAll(".link")
+			.data(peerings)
+			.enter()
+			.append("line")
+			.attr("class", "link")
+			.style("stroke-width", 3);
+
+var node = svg.selectAll('.node')
+			.data(asdata)
+			.enter()
+			.append('circle')
+			.attr('class', 'node')
+			.attr('r', 10)
+			.call(force.drag);
+
+node.append('text')
+	.text("maunz");
+
+force.on("tick", function() {
+	node.attr('cx', function(d) { return d.x; })
+		.attr('cy', function(d) { return d.y; });
+
+	link.attr("x1", function(d) { return d.source.x; })
+		.attr("y1", function(d) { return d.source.y; })
+		.attr("x2", function(d) { return d.target.x; })
+		.attr("y2", function(d) { return d.target.y; });
+});
+
+force.start();
+
+//var node = svg.selectAll(".node").data(nodes).enter().append("circle");
+//node.append("title").text(function(d) { return d.name });
+</script>
+
+
+<p>
+{% for AS in ASses %}
+	{{ AS.number }}<br />
+{% endfor %}
+</p>
+{% endblock %}

bgpdata/templates/bgpdata/map-example.html

@@ -0,0 +1,64 @@
+{% extends "base.html" %}
+{% block head %}
+<script src="http://d3js.org/d3.v3.min.js" charset="utf-8"></script>
+{% endblock %}
+{% block body %}
+
+<h3>Crawl run {{crawl.pk}} from {{crawl.startTime}}</h3>
+
+<script type="text/javascript" charset="utf-8">
+
+var data = [{% for AS in ASses%}{{AS.number}}{%if not forloop.last%},{%endif%}{%endfor%}];
+var width = 960,
+    height = 500;
+
+var color = d3.scale.category20();
+
+var force = d3.layout.force()
+    .charge(-120)
+    .linkDistance(30)
+    .size([width, height]);
+
+var svg = d3.select("body").append("svg")
+    .attr("width", width)
+    .attr("height", height);
+
+d3.json("miserables.json", function(error, graph) {
+  force
+      .nodes(graph.nodes)
+      .links(graph.links)
+      .start();
+
+  var link = svg.selectAll(".link")
+      .data(graph.links)
+    .enter().append("line")
+      .attr("class", "link")
+      .style("stroke-width", function(d) { return Math.sqrt(d.value); });
+
+  var node = svg.selectAll(".node")
+      .data(graph.nodes)
+    .enter().append("circle")
+      .attr("class", "node")
+      .attr("r", 5)
+      .style("fill", function(d) { return color(d.group); })
+      .call(force.drag);
+
+  node.append("title")
+      .text(function(d) { return d.name; });
+
+  force.on("tick", function() {
+    link.attr("x1", function(d) { return d.source.x; })
+        .attr("y1", function(d) { return d.source.y; })
+        .attr("x2", function(d) { return d.target.x; })
+        .attr("y2", function(d) { return d.target.y; });
+
+    node.attr("cx", function(d) { return d.x; })
+        .attr("cy", function(d) { return d.y; });
+  });
+});
+</script>
+
+{% for AS in ASses %}
+	{{ AS.number }}<br />
+{% endfor %}
+{% endblock %}

bgpdata/templates/bgpdata/map.html

@@ -0,0 +1,142 @@
+{% extends "base.html" %}
+{% block head %}
+{% load static from staticfiles %}
+<script src="{% static "js/d3.js" %}" charset="utf-8"></script>
+<style>
+.node {
+  stroke: #fff;
+  stroke-width: 1.5px;
+}
+
+.link {
+  stroke: #999;
+  stroke-opacity: .6;
+}
+</style>
+{% endblock %}
+{% block body %}
+
+<h3>Crawl run {{crawl.pk}} from {{crawl.startTime}}</h3>
+
+<div id="plotwin"></div>
+<script type="text/javascript" charset="utf-8">
+
+var asdata = [
+{% for AS in ASses %}
+	{id: {{AS.pk}}, nodetype: "AS", asnumber: {{AS.number}}, label: "{{AS.number}}", neighbors: {{AS.getPeerings.count}}, crawled: {%if AS.directlyCrawled%}true{%else%}false{%endif%}, online: {%if AS.online%}true{%else%}false{%endif%}}{%if not forloop.last%},{%endif%}
+{%endfor%}
+];
+
+var asdict = {
+{% for AS in ASses %}
+	{{AS.number}}: {{forloop.counter0}}{%if not forloop.last%},{%endif%}
+{% endfor %}
+};
+
+var peerings = [
+{% for p in peerings %}
+	{
+		source: asdict[{{p.as1.number}}],
+		target: asdict[{{p.as2.number}}],
+
+		id: {{p.pk}},
+		as1id: {{p.as1.id}},
+		as1number: {{p.as1.number}},
+		as2id: {{p.as2.id}},
+		as2number: {{p.as2.number}},
+
+		origin: "{{p.get_origin_display}}"
+	}{%if not forloop.last%},{%endif%}
+{%endfor%}
+];
+
+
+var width = 960,
+    height = 800;
+
+var svg = d3.select('#plotwin')
+			.append('svg')
+			.attr('width', width)
+			.attr('height', height);
+
+console.log(asdata);
+var force = d3.layout.force()
+	.nodes(asdata)
+	.links(peerings)
+    .charge(-500)
+//	.chargeDistance(300)
+//    .linkDistance(70)
+	.linkStrength(0.65)
+	.linkDistance(function(l) {
+		console.log(l);
+		neighs = Math.min(l.source.neighbors, l.target.neighbors);
+		console.log(neighs, "neighbors");
+		switch(neighs) {
+			case 0: return 40;
+			case 1: 
+			case 2: return 120;
+			case 3:
+			case 4: return 200;
+			default: return 250;
+		}
+	})
+    .size([width, height])
+	.start();
+
+
+var link = svg.selectAll(".link")
+			.data(peerings)
+			.enter()
+			.append("line")
+			.attr("class", "link")
+			.style("stroke-width", function(l) {
+				neighs = Math.min(l.source.neighbors, l.target.neighbors);
+				return 1 + Math.min(5, neighs);
+			});
+			//.style("stroke-width", 3);
+
+var node = svg.selectAll('.node')
+			.data(asdata)
+			.enter()
+			.append("g")
+			.call(force.drag);
+
+node.append("ellipse")
+	.attr("rx", 40)
+	.attr("ry", 20)
+	.attr("fill", function(d) {
+		if(d.crawled)
+			return "#94FF70";
+		else if(d.online)
+			return "#D1FFC2";
+			// return "#F0FFEB";
+		else
+			return "#FFCCCC";
+	})
+	.attr("stroke", "black")
+	.attr("stroke-width", "1px");
+
+node.append('text')
+	.attr("font-family", "sans-serif")
+	.attr("font-size", "13px")
+	.attr("font-weight", "bold")
+	.attr("dy", "4")
+	.attr("text-anchor", "middle")
+	.text(function(d) { return d.label; });
+
+force.on("tick", function() {
+	//node.attr('cx', function(d) { return d.x; })
+	//	.attr('cy', function(d) { return d.y; });
+	node.attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; });
+
+	link.attr("x1", function(d) { return d.source.x; })
+		.attr("y1", function(d) { return d.source.y; })
+		.attr("x2", function(d) { return d.target.x; })
+		.attr("y2", function(d) { return d.target.y; });
+});
+
+force.start();
+
+</script>
+
+{% endblock %}

bgpdata/templates/bgpdata/overview.html

@@ -0,0 +1,9 @@
+{% extends "base.html" %}
+{% block body %}
+<h3>Crawl results</h3>
+
+{% for crawl in crawls %}
+	<a href="/map/{{crawl.id}}/">Crawl {{crawl.id}} from {{crawl.startTime}}</a><br />
+{% endfor %}
+
+{% endblock %}

bgpdata/urls.py

@@ -0,0 +1,17 @@
+from django.conf.urls import patterns, url, include
+#from django.views.generic import RedirectView
+#from api import ASResource, CrawlResource
+
+#asResource = ASResource()
+#crawlResource = CrawlResource()
+
+urlpatterns = patterns('',
+	url(r'^$', 'bgpdata.views.overview'),
+	url(r'^([0-9]+)/$', 'bgpdata.views.showMap'),
+
+	#url(r'^api/crawl/(?P<crawlID>\d+)/asses/$', 'bgpdata.api.asses'),
+	#(r'^api/', include(asResource.urls)),
+	#(r'^api/', include(asResource.urls)),
+	#(r'^api/', include(crawlResource.urls)),
+)
+

bgpdata/views.py

@@ -1,3 +1,13 @@
 from django.shortcuts import render
+from bgpdata.models import CrawlRun, AS, Peering
 
-# Create your views here.
+def overview(request):
+	crawls = CrawlRun.objects.order_by("-startTime")
+	return render(request, 'bgpdata/overview.html', {"crawls": crawls})
+
+def showMap(request, crawlId):
+	crawl = CrawlRun.objects.get(id=crawlId)
+	ASses = AS.objects.filter(crawl=crawl)
+	peerings = Peering.objects.filter(as1__crawl=crawl)
+
+	return render(request, 'bgpdata/map.html', {"crawl": crawl, 'ASses': ASses, 'peerings': peerings})

bin/crawl.py

@@ -9,19 +9,20 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dnmapper.settings")
 import django
 django.setup()
 
+import datetime
 from django.utils import timezone
-from django.db.models import Q
+from django.db.models import Q, Max
 
 from bgpdata.models import ConfigHost, CrawlRun, CrawlLog, AS, BorderRouter, Announcement, Peering, BorderRouterPair
 from routerparsers import getBGPData, RouterParserException
 
 
-def getOrCreateAS(crawl, number):
+def getOrCreateAS(crawl, number, online=True):
 	currAS = None
 	try:
 		currAS = AS.objects.get(crawl=crawl, number=number)
 	except AS.DoesNotExist:
-		currAS = AS(crawl=crawl, number=number)
+		currAS = AS(crawl=crawl, number=number, online=online)
 		currAS.save()
 
 	return currAS
@@ -55,6 +56,9 @@ def main():
 		currASno = int(data["local_as"])
 		currAS = getOrCreateAS(crawl, currASno)
 
+		currAS.directlyCrawled = True
+		currAS.save()
+
 		currRouter = None
 		try:
 			currRouter = BorderRouter.objects.get(AS=currAS, routerID=data["local_id"])
@@ -115,10 +119,14 @@ def main():
 				print(" ---->", route["prefix"])
 				if "/" not in route["prefix"]:
 					continue
+
+				originAS = currAS
+				if len(route["path"]) > 0:
+					originAS = getOrCreateAS(crawl, route["path"][0])
 				ip, prefix = route["prefix"].split("/")
 				a = Announcement(router=currRouter, ip=ip, prefix=prefix,
 									ASPath=" ".join(route["path"]), nextHop=route["nexthop"],
-									originAS=currAS)
+									originAS=originAS)
 				a.save()
 		else:
 			print(" !! No routes found in host output")
@@ -144,6 +152,36 @@ def main():
 
 				firstAS = secondAS
 
+	# 3.2 add ASses, routers and peerings from old crawlruns (last should suffice)
+	# find 
+	print(" --> copy old ASses")
+	timerangeStart = crawl.startTime - datetime.timedelta(7)
+	oldASses = AS.objects.filter(crawl__startTime__gte=timerangeStart).values("number").annotate(lastSeen=Max('crawl_id')).filter(~Q(lastSeen=crawl.pk))
+
+	# 3.2.1. copy old asses
+	print(" ----> create ASses")
+	for oldASdata in oldASses:
+		print(" ------> AS", oldASdata["number"])
+		oldAS = AS.objects.get(number=oldASdata["number"], crawl=oldASdata["lastSeen"])
+
+		newAS = AS(number=oldAS.number, crawl=crawl, lastSeen=oldAS.crawl, directlyCrawled=False, online=False)
+		newAS.save()
+
+	# 3.2.2 copy peerings between old asses
+	print(" ----> copy peerings")
+	for oldASdata in oldASses:
+		print(" ------> AS", oldASdata["number"])
+		oldAS = AS.objects.get(number=oldASdata["number"], crawl=oldASdata["lastSeen"])
+		for peering in oldAS.getPeerings():
+			print(" --------> Peering %s <--> %s" % (peering.as1.number, peering.as2.number))
+			peering = Peering(
+						as1=AS.objects.get(number=peering.as1.number, crawl=crawl), 
+						as2=AS.objects.get(number=peering.as2.number, crawl=crawl),
+						origin=peering.origin)
+			peering.save()
+
+	# 3.3 FIXME: do we also want to have old peerings which do not exist anymore?
+
 	# 4. end crawl run
 	crawl.endTime = timezone.now()
 	crawl.save()

dnmapper/settings.py

@@ -26,6 +26,9 @@ TEMPLATE_DEBUG = True
 
 ALLOWED_HOSTS = []
 
+STATICFILES_DIRS = (
+	'static/',
+)
 
 # Application definition
 
@@ -37,8 +40,11 @@ INSTALLED_APPS = (
     'django.contrib.messages',
     'django.contrib.staticfiles',
 	'bgpdata',
+#	'tastypie',
 )
 
+API_LIMIT_PER_PAGE = 100
+
 MIDDLEWARE_CLASSES = (
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.common.CommonMiddleware',
@@ -76,6 +82,9 @@ USE_I18N = True
 USE_L10N = True
 
 USE_TZ = True
+TEMPLATE_DIRS = (
+	'templates/',
+)
 
 
 # Static files (CSS, JavaScript, Images)

dnmapper/urls.py

@@ -1,10 +1,14 @@
 from django.conf.urls import patterns, include, url
 from django.contrib import admin
+from django.views.generic import RedirectView
+import bgpdata.urls
 
 urlpatterns = patterns('',
     # Examples:
     # url(r'^$', 'dnmapper.views.home', name='home'),
     # url(r'^blog/', include('blog.urls')),
+	url(r'^$', RedirectView.as_view(url='/map/')),
+	url(r'^map/', include(bgpdata.urls)),
 
-    url(r'^admin/', include(admin.site.urls)),
+	url(r'^admin/', include(admin.site.urls)),
 )

static/js/LICENSE

@@ -0,0 +1,26 @@
+Copyright (c) 2010-2015, Michael Bostock
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* The name Michael Bostock may not be used to endorse or promote products
+  derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL MICHAEL BOSTOCK BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

templates/base.html

@@ -0,0 +1,10 @@
+<!doctype html5>
+<html>
+<head>
+{% block head %}{% endblock %}
+</head>
+<body>
+<h1>DarkMap</h1>
+{% block body %}{% endblock %}
+</body>
+</html>

templates/templ.html

@@ -0,0 +1,3 @@
+{% extends "base.html" %}
+{% block body %}
+{% endblock %}