seba
/
dnmapper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

working map

This commit is contained in:
Sebastian Lohff 2015-03-27 15:33:54 +01:00
parent 12dfa7f3b4
commit ff61388de5
18 changed files with 10038 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
bgpdata/api.py Normal file
View File

@ -0,0 +1,19 @@
#from tastypie.resources import ModelResource, ALL_WITH_RELATIONS
#from tastypie import fields
#from bgpdata.models import AS, CrawlRun
#
#class ASResource(ModelResource):
# crawl = fields.ForeignKey("bgpdata.api.CrawlResource", "crawl")
# class Meta:
# list_allowed_methods = ['get']
# detail_allowed_methods = ['get']
# filtering = {'crawl': ALL_WITH_RELATIONS}
#
# queryset = AS.objects.all()
# resource_name = "as"
#
#class CrawlResource(ModelResource):
# class Meta:
# queryset = CrawlRun.objects.all()
# resource_name = "crawl"

27
bgpdata/migrations/0009_auto_20150326_2207.py Normal file
View File

@ -0,0 +1,27 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('bgpdata', '0008_auto_20150322_1906'),
]
operations = [
migrations.AddField(
model_name='as',
name='directlyCrawled',
field=models.BooleanField(default=False),
preserve_default=True,
),
migrations.AlterField(
model_name='crawllog',
name='host',
field=models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, blank=True, to='bgpdata.ConfigHost', null=True),
preserve_default=True,
),
]

26
bgpdata/migrations/0010_auto_20150327_0003.py Normal file
View File

@ -0,0 +1,26 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('bgpdata', '0009_auto_20150326_2207'),
]
operations = [
migrations.AddField(
model_name='as',
name='lastSeen',
field=models.ForeignKey(related_name='as_lastseen', default=None, blank=True, to='bgpdata.CrawlRun', null=True),
preserve_default=True,
),
migrations.AddField(
model_name='as',
name='online',
field=models.BooleanField(default=True),
preserve_default=True,
),
]

6
bgpdata/models.py
View File

@ -39,7 +39,7 @@ class CrawlLog(models.Model):
)
crawl = models.ForeignKey(CrawlRun)
host = models.ForeignKey(ConfigHost, null=True, blank=True)
host = models.ForeignKey(ConfigHost, null=True, blank=True, on_delete=models.SET_NULL)
logtime = models.DateTimeField(auto_now_add=True)
severity = models.CharField(max_length=10, choices=SEVERITY)
message = models.TextField()
@ -65,6 +65,10 @@ class AS(models.Model):
crawl = models.ForeignKey(CrawlRun)
number = models.IntegerField()
directlyCrawled = models.BooleanField(default=False)
online = models.BooleanField(default=True)
lastSeen = models.ForeignKey(CrawlRun, blank=True, null=True, default=None, related_name='as_lastseen')
def __unicode__(self):
return u"AS %s (crawl %d)" % (self.number, self.crawl.pk)

114
bgpdata/templates/bgpdata/map-2.html Normal file
View File

@ -0,0 +1,114 @@
{% extends "base.html" %}
{% block head %}
{% load static from staticfiles %}
<script src="{% static "js/d3.js" %}" charset="utf-8"></script>
<style>
.node {
stroke: #fff;
stroke-width: 1.5px;
}
.link {
stroke: #999;
stroke-opacity: .6;
}
</style>
{% endblock %}
{% block body %}
<h3>Crawl run {{crawl.pk}} from {{crawl.startTime}}</h3>
<div id="plotwin"></div>
<script type="text/javascript" charset="utf-8">
var asdata = [
{% for AS in ASses %}
{id: {{AS.pk}}, numer: {{AS.number}}, label: 'MAUNZ'}{%if not forloop.last%},{%endif%}
{%endfor%}
];
var asdict = {
{% for AS in ASses %}
{{AS.number}}: {{forloop.counter0}}{%if not forloop.last%},{%endif%}
{% endfor %}
};
var peerings = [
{% for p in peerings %}
{
source: asdict[{{p.as1.number}}],
target: asdict[{{p.as2.number}}],
id: {{p.pk}},
as1id: {{p.as1.id}},
as1number: {{p.as1.number}},
as2id: {{p.as2.id}},
as2number: {{p.as2.number}},
origin: "{{p.get_origin_display}}"
}{%if not forloop.last%},{%endif%}
{%endfor%}
];
var width = 960,
height = 600;
var svg = d3.select('#plotwin')
.append('svg')
.attr('width', width)
.attr('height', height);
console.log(asdata);
var force = d3.layout.force()
.nodes(asdata)
.links(peerings)
.charge(-200)
.linkDistance(70)
.size([width, height])
.start();
// .charge(-120)
// .linkDistance(30)
var link = svg.selectAll(".link")
.data(peerings)
.enter()
.append("line")
.attr("class", "link")
.style("stroke-width", 3);
var node = svg.selectAll('.node')
.data(asdata)
.enter()
.append('circle')
.attr('class', 'node')
.attr('r', 10)
.call(force.drag);
node.append('text')
.text("maunz");
force.on("tick", function() {
node.attr('cx', function(d) { return d.x; })
.attr('cy', function(d) { return d.y; });
link.attr("x1", function(d) { return d.source.x; })
.attr("y1", function(d) { return d.source.y; })
.attr("x2", function(d) { return d.target.x; })
.attr("y2", function(d) { return d.target.y; });
});
force.start();
//var node = svg.selectAll(".node").data(nodes).enter().append("circle");
//node.append("title").text(function(d) { return d.name });
</script>
<p>
{% for AS in ASses %}
{{ AS.number }}<br />
{% endfor %}
</p>
{% endblock %}

64
bgpdata/templates/bgpdata/map-example.html Normal file
View File

@ -0,0 +1,64 @@
{% extends "base.html" %}
{% block head %}
<script src="http://d3js.org/d3.v3.min.js" charset="utf-8"></script>
{% endblock %}
{% block body %}
<h3>Crawl run {{crawl.pk}} from {{crawl.startTime}}</h3>
<script type="text/javascript" charset="utf-8">
var data = [{% for AS in ASses%}{{AS.number}}{%if not forloop.last%},{%endif%}{%endfor%}];
var width = 960,
height = 500;
var color = d3.scale.category20();
var force = d3.layout.force()
.charge(-120)
.linkDistance(30)
.size([width, height]);
var svg = d3.select("body").append("svg")
.attr("width", width)
.attr("height", height);
d3.json("miserables.json", function(error, graph) {
force
.nodes(graph.nodes)
.links(graph.links)
.start();
var link = svg.selectAll(".link")
.data(graph.links)
.enter().append("line")
.attr("class", "link")
.style("stroke-width", function(d) { return Math.sqrt(d.value); });
var node = svg.selectAll(".node")
.data(graph.nodes)
.enter().append("circle")
.attr("class", "node")
.attr("r", 5)
.style("fill", function(d) { return color(d.group); })
.call(force.drag);
node.append("title")
.text(function(d) { return d.name; });
force.on("tick", function() {
link.attr("x1", function(d) { return d.source.x; })
.attr("y1", function(d) { return d.source.y; })
.attr("x2", function(d) { return d.target.x; })
.attr("y2", function(d) { return d.target.y; });
node.attr("cx", function(d) { return d.x; })
.attr("cy", function(d) { return d.y; });
});
});
</script>
{% for AS in ASses %}
{{ AS.number }}<br />
{% endfor %}
{% endblock %}

142
bgpdata/templates/bgpdata/map.html Normal file
View File

@ -0,0 +1,142 @@
{% extends "base.html" %}
{% block head %}
{% load static from staticfiles %}
<script src="{% static "js/d3.js" %}" charset="utf-8"></script>
<style>
.node {
stroke: #fff;
stroke-width: 1.5px;
}
.link {
stroke: #999;
stroke-opacity: .6;
}
</style>
{% endblock %}
{% block body %}
<h3>Crawl run {{crawl.pk}} from {{crawl.startTime}}</h3>
<div id="plotwin"></div>
<script type="text/javascript" charset="utf-8">
var asdata = [
{% for AS in ASses %}
{id: {{AS.pk}}, nodetype: "AS", asnumber: {{AS.number}}, label: "{{AS.number}}", neighbors: {{AS.getPeerings.count}}, crawled: {%if AS.directlyCrawled%}true{%else%}false{%endif%}, online: {%if AS.online%}true{%else%}false{%endif%}}{%if not forloop.last%},{%endif%}
{%endfor%}
];
var asdict = {
{% for AS in ASses %}
{{AS.number}}: {{forloop.counter0}}{%if not forloop.last%},{%endif%}
{% endfor %}
};
var peerings = [
{% for p in peerings %}
{
source: asdict[{{p.as1.number}}],
target: asdict[{{p.as2.number}}],
id: {{p.pk}},
as1id: {{p.as1.id}},
as1number: {{p.as1.number}},
as2id: {{p.as2.id}},
as2number: {{p.as2.number}},
origin: "{{p.get_origin_display}}"
}{%if not forloop.last%},{%endif%}
{%endfor%}
];
var width = 960,
height = 800;
var svg = d3.select('#plotwin')
.append('svg')
.attr('width', width)
.attr('height', height);
console.log(asdata);
var force = d3.layout.force()
.nodes(asdata)
.links(peerings)
.charge(-500)
// .chargeDistance(300)
// .linkDistance(70)
.linkStrength(0.65)
.linkDistance(function(l) {
console.log(l);
neighs = Math.min(l.source.neighbors, l.target.neighbors);
console.log(neighs, "neighbors");
switch(neighs) {
case 0: return 40;
case 1:
case 2: return 120;
case 3:
case 4: return 200;
default: return 250;
}
})
.size([width, height])
.start();
var link = svg.selectAll(".link")
.data(peerings)
.enter()
.append("line")
.attr("class", "link")
.style("stroke-width", function(l) {
neighs = Math.min(l.source.neighbors, l.target.neighbors);
return 1 + Math.min(5, neighs);
});
//.style("stroke-width", 3);
var node = svg.selectAll('.node')
.data(asdata)
.enter()
.append("g")
.call(force.drag);
node.append("ellipse")
.attr("rx", 40)
.attr("ry", 20)
.attr("fill", function(d) {
if(d.crawled)
return "#94FF70";
else if(d.online)
return "#D1FFC2";
// return "#F0FFEB";
else
return "#FFCCCC";
})
.attr("stroke", "black")
.attr("stroke-width", "1px");
node.append('text')
.attr("font-family", "sans-serif")
.attr("font-size", "13px")
.attr("font-weight", "bold")
.attr("dy", "4")
.attr("text-anchor", "middle")
.text(function(d) { return d.label; });
force.on("tick", function() {
//node.attr('cx', function(d) { return d.x; })
// .attr('cy', function(d) { return d.y; });
node.attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; });
link.attr("x1", function(d) { return d.source.x; })
.attr("y1", function(d) { return d.source.y; })
.attr("x2", function(d) { return d.target.x; })
.attr("y2", function(d) { return d.target.y; });
});
force.start();
</script>
{% endblock %}

9
bgpdata/templates/bgpdata/overview.html Normal file
View File

@ -0,0 +1,9 @@
{% extends "base.html" %}
{% block body %}
<h3>Crawl results</h3>
{% for crawl in crawls %}
<a href="/map/{{crawl.id}}/">Crawl {{crawl.id}} from {{crawl.startTime}}</a><br />
{% endfor %}
{% endblock %}

17
bgpdata/urls.py Normal file
View File

@ -0,0 +1,17 @@
from django.conf.urls import patterns, url, include
#from django.views.generic import RedirectView
#from api import ASResource, CrawlResource
#asResource = ASResource()
#crawlResource = CrawlResource()
urlpatterns = patterns('',
url(r'^$', 'bgpdata.views.overview'),
url(r'^([0-9]+)/$', 'bgpdata.views.showMap'),
#url(r'^api/crawl/(?P<crawlID>\d+)/asses/$', 'bgpdata.api.asses'),
#(r'^api/', include(asResource.urls)),
#(r'^api/', include(asResource.urls)),
#(r'^api/', include(crawlResource.urls)),
)

12
bgpdata/views.py
View File

@ -1,3 +1,13 @@
from django.shortcuts import render
from bgpdata.models import CrawlRun, AS, Peering
# Create your views here.
def overview(request):
crawls = CrawlRun.objects.order_by("-startTime")
return render(request, 'bgpdata/overview.html', {"crawls": crawls})
def showMap(request, crawlId):
crawl = CrawlRun.objects.get(id=crawlId)
ASses = AS.objects.filter(crawl=crawl)
peerings = Peering.objects.filter(as1__crawl=crawl)
return render(request, 'bgpdata/map.html', {"crawl": crawl, 'ASses': ASses, 'peerings': peerings})

46
bin/crawl.py
View File

@ -9,19 +9,20 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dnmapper.settings")
import django
django.setup()
import datetime
from django.utils import timezone
from django.db.models import Q
from django.db.models import Q, Max
from bgpdata.models import ConfigHost, CrawlRun, CrawlLog, AS, BorderRouter, Announcement, Peering, BorderRouterPair
from routerparsers import getBGPData, RouterParserException
def getOrCreateAS(crawl, number):
def getOrCreateAS(crawl, number, online=True):
currAS = None
try:
currAS = AS.objects.get(crawl=crawl, number=number)
except AS.DoesNotExist:
currAS = AS(crawl=crawl, number=number)
currAS = AS(crawl=crawl, number=number, online=online)
currAS.save()
return currAS
@ -55,6 +56,9 @@ def main():
currASno = int(data["local_as"])
currAS = getOrCreateAS(crawl, currASno)
currAS.directlyCrawled = True
currAS.save()
currRouter = None
try:
currRouter = BorderRouter.objects.get(AS=currAS, routerID=data["local_id"])
@ -115,10 +119,14 @@ def main():
print(" ---->", route["prefix"])
if "/" not in route["prefix"]:
continue
originAS = currAS
if len(route["path"]) > 0:
originAS = getOrCreateAS(crawl, route["path"][0])
ip, prefix = route["prefix"].split("/")
a = Announcement(router=currRouter, ip=ip, prefix=prefix,
ASPath=" ".join(route["path"]), nextHop=route["nexthop"],
originAS=currAS)
originAS=originAS)
a.save()
else:
print(" !! No routes found in host output")
@ -144,6 +152,36 @@ def main():
firstAS = secondAS
# 3.2 add ASses, routers and peerings from old crawlruns (last should suffice)
# find
print(" --> copy old ASses")
timerangeStart = crawl.startTime - datetime.timedelta(7)
oldASses = AS.objects.filter(crawl__startTime__gte=timerangeStart).values("number").annotate(lastSeen=Max('crawl_id')).filter(~Q(lastSeen=crawl.pk))
# 3.2.1. copy old asses
print(" ----> create ASses")
for oldASdata in oldASses:
print(" ------> AS", oldASdata["number"])
oldAS = AS.objects.get(number=oldASdata["number"], crawl=oldASdata["lastSeen"])
newAS = AS(number=oldAS.number, crawl=crawl, lastSeen=oldAS.crawl, directlyCrawled=False, online=False)
newAS.save()
# 3.2.2 copy peerings between old asses
print(" ----> copy peerings")
for oldASdata in oldASses:
print(" ------> AS", oldASdata["number"])
oldAS = AS.objects.get(number=oldASdata["number"], crawl=oldASdata["lastSeen"])
for peering in oldAS.getPeerings():
print(" --------> Peering %s <--> %s" % (peering.as1.number, peering.as2.number))
peering = Peering(
as1=AS.objects.get(number=peering.as1.number, crawl=crawl),
as2=AS.objects.get(number=peering.as2.number, crawl=crawl),
origin=peering.origin)
peering.save()
# 3.3 FIXME: do we also want to have old peerings which do not exist anymore?
# 4. end crawl run
crawl.endTime = timezone.now()
crawl.save()

9
dnmapper/settings.py
View File

@ -26,6 +26,9 @@ TEMPLATE_DEBUG = True
ALLOWED_HOSTS = []
STATICFILES_DIRS = (
'static/',
)
# Application definition
@ -37,8 +40,11 @@ INSTALLED_APPS = (
'django.contrib.messages',
'django.contrib.staticfiles',
'bgpdata',
# 'tastypie',
)
API_LIMIT_PER_PAGE = 100
MIDDLEWARE_CLASSES = (
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
@ -76,6 +82,9 @@ USE_I18N = True
USE_L10N = True
USE_TZ = True
TEMPLATE_DIRS = (
'templates/',
)
# Static files (CSS, JavaScript, Images)

6
dnmapper/urls.py
View File

@ -1,10 +1,14 @@
from django.conf.urls import patterns, include, url
from django.contrib import admin
from django.views.generic import RedirectView
import bgpdata.urls
urlpatterns = patterns('',
# Examples:
# url(r'^$', 'dnmapper.views.home', name='home'),
# url(r'^blog/', include('blog.urls')),
url(r'^$', RedirectView.as_view(url='/map/')),
url(r'^map/', include(bgpdata.urls)),
url(r'^admin/', include(admin.site.urls)),
url(r'^admin/', include(admin.site.urls)),
)

26
static/js/LICENSE Normal file
View File

@ -0,0 +1,26 @@
Copyright (c) 2010-2015, Michael Bostock
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* The name Michael Bostock may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL MICHAEL BOSTOCK BE LIABLE FOR ANY DIRECT,
INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

9504
static/js/d3.js vendored Normal file
View File

File diff suppressed because it is too large Load Diff

5
static/js/d3.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

10
templates/base.html Normal file
View File

@ -0,0 +1,10 @@
<!doctype html5>
<html>
<head>
{% block head %}{% endblock %}
</head>
<body>
<h1>DarkMap</h1>
{% block body %}{% endblock %}
</body>
</html>

3
templates/templ.html Normal file
View File

@ -0,0 +1,3 @@
{% extends "base.html" %}
{% block body %}
{% endblock %}