Browse Source

working map

master
Sebastian Lohff 7 years ago
parent
commit
ff61388de5
  1. 19
      bgpdata/api.py
  2. 27
      bgpdata/migrations/0009_auto_20150326_2207.py
  3. 26
      bgpdata/migrations/0010_auto_20150327_0003.py
  4. 6
      bgpdata/models.py
  5. 114
      bgpdata/templates/bgpdata/map-2.html
  6. 64
      bgpdata/templates/bgpdata/map-example.html
  7. 142
      bgpdata/templates/bgpdata/map.html
  8. 9
      bgpdata/templates/bgpdata/overview.html
  9. 17
      bgpdata/urls.py
  10. 12
      bgpdata/views.py
  11. 46
      bin/crawl.py
  12. 9
      dnmapper/settings.py
  13. 6
      dnmapper/urls.py
  14. 26
      static/js/LICENSE
  15. 9504
      static/js/d3.js
  16. 5
      static/js/d3.min.js
  17. 10
      templates/base.html
  18. 3
      templates/templ.html

19
bgpdata/api.py

@ -0,0 +1,19 @@ @@ -0,0 +1,19 @@
#from tastypie.resources import ModelResource, ALL_WITH_RELATIONS
#from tastypie import fields
#from bgpdata.models import AS, CrawlRun
#
#class ASResource(ModelResource):
# crawl = fields.ForeignKey("bgpdata.api.CrawlResource", "crawl")
# class Meta:
# list_allowed_methods = ['get']
# detail_allowed_methods = ['get']
# filtering = {'crawl': ALL_WITH_RELATIONS}
#
# queryset = AS.objects.all()
# resource_name = "as"
#
#class CrawlResource(ModelResource):
# class Meta:
# queryset = CrawlRun.objects.all()
# resource_name = "crawl"

27
bgpdata/migrations/0009_auto_20150326_2207.py

@ -0,0 +1,27 @@ @@ -0,0 +1,27 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('bgpdata', '0008_auto_20150322_1906'),
]
operations = [
migrations.AddField(
model_name='as',
name='directlyCrawled',
field=models.BooleanField(default=False),
preserve_default=True,
),
migrations.AlterField(
model_name='crawllog',
name='host',
field=models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, blank=True, to='bgpdata.ConfigHost', null=True),
preserve_default=True,
),
]

26
bgpdata/migrations/0010_auto_20150327_0003.py

@ -0,0 +1,26 @@ @@ -0,0 +1,26 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('bgpdata', '0009_auto_20150326_2207'),
]
operations = [
migrations.AddField(
model_name='as',
name='lastSeen',
field=models.ForeignKey(related_name='as_lastseen', default=None, blank=True, to='bgpdata.CrawlRun', null=True),
preserve_default=True,
),
migrations.AddField(
model_name='as',
name='online',
field=models.BooleanField(default=True),
preserve_default=True,
),
]

6
bgpdata/models.py

@ -39,7 +39,7 @@ class CrawlLog(models.Model): @@ -39,7 +39,7 @@ class CrawlLog(models.Model):
)
crawl = models.ForeignKey(CrawlRun)
host = models.ForeignKey(ConfigHost, null=True, blank=True)
host = models.ForeignKey(ConfigHost, null=True, blank=True, on_delete=models.SET_NULL)
logtime = models.DateTimeField(auto_now_add=True)
severity = models.CharField(max_length=10, choices=SEVERITY)
message = models.TextField()
@ -65,6 +65,10 @@ class AS(models.Model): @@ -65,6 +65,10 @@ class AS(models.Model):
crawl = models.ForeignKey(CrawlRun)
number = models.IntegerField()
directlyCrawled = models.BooleanField(default=False)
online = models.BooleanField(default=True)
lastSeen = models.ForeignKey(CrawlRun, blank=True, null=True, default=None, related_name='as_lastseen')
def __unicode__(self):
return u"AS %s (crawl %d)" % (self.number, self.crawl.pk)

114
bgpdata/templates/bgpdata/map-2.html

@ -0,0 +1,114 @@ @@ -0,0 +1,114 @@
{% extends "base.html" %}
{% block head %}
{% load static from staticfiles %}
<script src="{% static "js/d3.js" %}" charset="utf-8"></script>
<style>
.node {
stroke: #fff;
stroke-width: 1.5px;
}
.link {
stroke: #999;
stroke-opacity: .6;
}
</style>
{% endblock %}
{% block body %}
<h3>Crawl run {{crawl.pk}} from {{crawl.startTime}}</h3>
<div id="plotwin"></div>
<script type="text/javascript" charset="utf-8">
var asdata = [
{% for AS in ASses %}
{id: {{AS.pk}}, numer: {{AS.number}}, label: 'MAUNZ'}{%if not forloop.last%},{%endif%}
{%endfor%}
];
var asdict = {
{% for AS in ASses %}
{{AS.number}}: {{forloop.counter0}}{%if not forloop.last%},{%endif%}
{% endfor %}
};
var peerings = [
{% for p in peerings %}
{
source: asdict[{{p.as1.number}}],
target: asdict[{{p.as2.number}}],
id: {{p.pk}},
as1id: {{p.as1.id}},
as1number: {{p.as1.number}},
as2id: {{p.as2.id}},
as2number: {{p.as2.number}},
origin: "{{p.get_origin_display}}"
}{%if not forloop.last%},{%endif%}
{%endfor%}
];
var width = 960,
height = 600;
var svg = d3.select('#plotwin')
.append('svg')
.attr('width', width)
.attr('height', height);
console.log(asdata);
var force = d3.layout.force()
.nodes(asdata)
.links(peerings)
.charge(-200)
.linkDistance(70)
.size([width, height])
.start();
// .charge(-120)
// .linkDistance(30)
var link = svg.selectAll(".link")
.data(peerings)
.enter()
.append("line")
.attr("class", "link")
.style("stroke-width", 3);
var node = svg.selectAll('.node')
.data(asdata)
.enter()
.append('circle')
.attr('class', 'node')
.attr('r', 10)
.call(force.drag);
node.append('text')
.text("maunz");
force.on("tick", function() {
node.attr('cx', function(d) { return d.x; })
.attr('cy', function(d) { return d.y; });
link.attr("x1", function(d) { return d.source.x; })
.attr("y1", function(d) { return d.source.y; })
.attr("x2", function(d) { return d.target.x; })
.attr("y2", function(d) { return d.target.y; });
});
force.start();
//var node = svg.selectAll(".node").data(nodes).enter().append("circle");
//node.append("title").text(function(d) { return d.name });
</script>
<p>
{% for AS in ASses %}
{{ AS.number }}<br />
{% endfor %}
</p>
{% endblock %}

64
bgpdata/templates/bgpdata/map-example.html

@ -0,0 +1,64 @@ @@ -0,0 +1,64 @@
{% extends "base.html" %}
{% block head %}
<script src="http://d3js.org/d3.v3.min.js" charset="utf-8"></script>
{% endblock %}
{% block body %}
<h3>Crawl run {{crawl.pk}} from {{crawl.startTime}}</h3>
<script type="text/javascript" charset="utf-8">
var data = [{% for AS in ASses%}{{AS.number}}{%if not forloop.last%},{%endif%}{%endfor%}];
var width = 960,
height = 500;
var color = d3.scale.category20();
var force = d3.layout.force()
.charge(-120)
.linkDistance(30)
.size([width, height]);
var svg = d3.select("body").append("svg")
.attr("width", width)
.attr("height", height);
d3.json("miserables.json", function(error, graph) {
force
.nodes(graph.nodes)
.links(graph.links)
.start();
var link = svg.selectAll(".link")
.data(graph.links)
.enter().append("line")
.attr("class", "link")
.style("stroke-width", function(d) { return Math.sqrt(d.value); });
var node = svg.selectAll(".node")
.data(graph.nodes)
.enter().append("circle")
.attr("class", "node")
.attr("r", 5)
.style("fill", function(d) { return color(d.group); })
.call(force.drag);
node.append("title")
.text(function(d) { return d.name; });
force.on("tick", function() {
link.attr("x1", function(d) { return d.source.x; })
.attr("y1", function(d) { return d.source.y; })
.attr("x2", function(d) { return d.target.x; })
.attr("y2", function(d) { return d.target.y; });
node.attr("cx", function(d) { return d.x; })
.attr("cy", function(d) { return d.y; });
});
});
</script>
{% for AS in ASses %}
{{ AS.number }}<br />
{% endfor %}
{% endblock %}

142
bgpdata/templates/bgpdata/map.html

@ -0,0 +1,142 @@ @@ -0,0 +1,142 @@
{% extends "base.html" %}
{% block head %}
{% load static from staticfiles %}
<script src="{% static "js/d3.js" %}" charset="utf-8"></script>
<style>
.node {
stroke: #fff;
stroke-width: 1.5px;
}
.link {
stroke: #999;
stroke-opacity: .6;
}
</style>
{% endblock %}
{% block body %}
<h3>Crawl run {{crawl.pk}} from {{crawl.startTime}}</h3>
<div id="plotwin"></div>
<script type="text/javascript" charset="utf-8">
var asdata = [
{% for AS in ASses %}
{id: {{AS.pk}}, nodetype: "AS", asnumber: {{AS.number}}, label: "{{AS.number}}", neighbors: {{AS.getPeerings.count}}, crawled: {%if AS.directlyCrawled%}true{%else%}false{%endif%}, online: {%if AS.online%}true{%else%}false{%endif%}}{%if not forloop.last%},{%endif%}
{%endfor%}
];
var asdict = {
{% for AS in ASses %}
{{AS.number}}: {{forloop.counter0}}{%if not forloop.last%},{%endif%}
{% endfor %}
};
var peerings = [
{% for p in peerings %}
{
source: asdict[{{p.as1.number}}],
target: asdict[{{p.as2.number}}],
id: {{p.pk}},
as1id: {{p.as1.id}},
as1number: {{p.as1.number}},
as2id: {{p.as2.id}},
as2number: {{p.as2.number}},
origin: "{{p.get_origin_display}}"
}{%if not forloop.last%},{%endif%}
{%endfor%}
];
var width = 960,
height = 800;
var svg = d3.select('#plotwin')
.append('svg')
.attr('width', width)
.attr('height', height);
console.log(asdata);
var force = d3.layout.force()
.nodes(asdata)
.links(peerings)
.charge(-500)
// .chargeDistance(300)
// .linkDistance(70)
.linkStrength(0.65)
.linkDistance(function(l) {
console.log(l);
neighs = Math.min(l.source.neighbors, l.target.neighbors);
console.log(neighs, "neighbors");
switch(neighs) {
case 0: return 40;
case 1:
case 2: return 120;
case 3:
case 4: return 200;
default: return 250;
}
})
.size([width, height])
.start();
var link = svg.selectAll(".link")
.data(peerings)
.enter()
.append("line")
.attr("class", "link")
.style("stroke-width", function(l) {
neighs = Math.min(l.source.neighbors, l.target.neighbors);
return 1 + Math.min(5, neighs);
});
//.style("stroke-width", 3);
var node = svg.selectAll('.node')
.data(asdata)
.enter()
.append("g")
.call(force.drag);
node.append("ellipse")
.attr("rx", 40)
.attr("ry", 20)
.attr("fill", function(d) {
if(d.crawled)
return "#94FF70";
else if(d.online)
return "#D1FFC2";
// return "#F0FFEB";
else
return "#FFCCCC";
})
.attr("stroke", "black")
.attr("stroke-width", "1px");
node.append('text')
.attr("font-family", "sans-serif")
.attr("font-size", "13px")
.attr("font-weight", "bold")
.attr("dy", "4")
.attr("text-anchor", "middle")
.text(function(d) { return d.label; });
force.on("tick", function() {
//node.attr('cx', function(d) { return d.x; })
// .attr('cy', function(d) { return d.y; });
node.attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; });
link.attr("x1", function(d) { return d.source.x; })
.attr("y1", function(d) { return d.source.y; })
.attr("x2", function(d) { return d.target.x; })
.attr("y2", function(d) { return d.target.y; });
});
force.start();
</script>
{% endblock %}

9
bgpdata/templates/bgpdata/overview.html

@ -0,0 +1,9 @@ @@ -0,0 +1,9 @@
{% extends "base.html" %}
{% block body %}
<h3>Crawl results</h3>
{% for crawl in crawls %}
<a href="/map/{{crawl.id}}/">Crawl {{crawl.id}} from {{crawl.startTime}}</a><br />
{% endfor %}
{% endblock %}

17
bgpdata/urls.py

@ -0,0 +1,17 @@ @@ -0,0 +1,17 @@
from django.conf.urls import patterns, url, include
#from django.views.generic import RedirectView
#from api import ASResource, CrawlResource
#asResource = ASResource()
#crawlResource = CrawlResource()
urlpatterns = patterns('',
url(r'^$', 'bgpdata.views.overview'),
url(r'^([0-9]+)/$', 'bgpdata.views.showMap'),
#url(r'^api/crawl/(?P<crawlID>\d+)/asses/$', 'bgpdata.api.asses'),
#(r'^api/', include(asResource.urls)),
#(r'^api/', include(asResource.urls)),
#(r'^api/', include(crawlResource.urls)),
)

12
bgpdata/views.py

@ -1,3 +1,13 @@ @@ -1,3 +1,13 @@
from django.shortcuts import render
from bgpdata.models import CrawlRun, AS, Peering
# Create your views here.
def overview(request):
crawls = CrawlRun.objects.order_by("-startTime")
return render(request, 'bgpdata/overview.html', {"crawls": crawls})
def showMap(request, crawlId):
crawl = CrawlRun.objects.get(id=crawlId)
ASses = AS.objects.filter(crawl=crawl)
peerings = Peering.objects.filter(as1__crawl=crawl)
return render(request, 'bgpdata/map.html', {"crawl": crawl, 'ASses': ASses, 'peerings': peerings})

46
bin/crawl.py

@ -9,19 +9,20 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dnmapper.settings") @@ -9,19 +9,20 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dnmapper.settings")
import django
django.setup()
import datetime
from django.utils import timezone
from django.db.models import Q
from django.db.models import Q, Max
from bgpdata.models import ConfigHost, CrawlRun, CrawlLog, AS, BorderRouter, Announcement, Peering, BorderRouterPair
from routerparsers import getBGPData, RouterParserException
def getOrCreateAS(crawl, number):
def getOrCreateAS(crawl, number, online=True):
currAS = None
try:
currAS = AS.objects.get(crawl=crawl, number=number)
except AS.DoesNotExist:
currAS = AS(crawl=crawl, number=number)
currAS = AS(crawl=crawl, number=number, online=online)
currAS.save()
return currAS
@ -55,6 +56,9 @@ def main(): @@ -55,6 +56,9 @@ def main():
currASno = int(data["local_as"])
currAS = getOrCreateAS(crawl, currASno)
currAS.directlyCrawled = True
currAS.save()
currRouter = None
try:
currRouter = BorderRouter.objects.get(AS=currAS, routerID=data["local_id"])
@ -115,10 +119,14 @@ def main(): @@ -115,10 +119,14 @@ def main():
print(" ---->", route["prefix"])
if "/" not in route["prefix"]:
continue
originAS = currAS
if len(route["path"]) > 0:
originAS = getOrCreateAS(crawl, route["path"][0])
ip, prefix = route["prefix"].split("/")
a = Announcement(router=currRouter, ip=ip, prefix=prefix,
ASPath=" ".join(route["path"]), nextHop=route["nexthop"],
originAS=currAS)
originAS=originAS)
a.save()
else:
print(" !! No routes found in host output")
@ -144,6 +152,36 @@ def main(): @@ -144,6 +152,36 @@ def main():
firstAS = secondAS
# 3.2 add ASses, routers and peerings from old crawlruns (last should suffice)
# find
print(" --> copy old ASses")
timerangeStart = crawl.startTime - datetime.timedelta(7)
oldASses = AS.objects.filter(crawl__startTime__gte=timerangeStart).values("number").annotate(lastSeen=Max('crawl_id')).filter(~Q(lastSeen=crawl.pk))
# 3.2.1. copy old asses
print(" ----> create ASses")
for oldASdata in oldASses:
print(" ------> AS", oldASdata["number"])
oldAS = AS.objects.get(number=oldASdata["number"], crawl=oldASdata["lastSeen"])
newAS = AS(number=oldAS.number, crawl=crawl, lastSeen=oldAS.crawl, directlyCrawled=False, online=False)
newAS.save()
# 3.2.2 copy peerings between old asses
print(" ----> copy peerings")
for oldASdata in oldASses:
print(" ------> AS", oldASdata["number"])
oldAS = AS.objects.get(number=oldASdata["number"], crawl=oldASdata["lastSeen"])
for peering in oldAS.getPeerings():
print(" --------> Peering %s <--> %s" % (peering.as1.number, peering.as2.number))
peering = Peering(
as1=AS.objects.get(number=peering.as1.number, crawl=crawl),
as2=AS.objects.get(number=peering.as2.number, crawl=crawl),
origin=peering.origin)
peering.save()
# 3.3 FIXME: do we also want to have old peerings which do not exist anymore?
# 4. end crawl run
crawl.endTime = timezone.now()
crawl.save()

9
dnmapper/settings.py

@ -26,6 +26,9 @@ TEMPLATE_DEBUG = True @@ -26,6 +26,9 @@ TEMPLATE_DEBUG = True
ALLOWED_HOSTS = []
STATICFILES_DIRS = (
'static/',
)
# Application definition
@ -37,8 +40,11 @@ INSTALLED_APPS = ( @@ -37,8 +40,11 @@ INSTALLED_APPS = (
'django.contrib.messages',
'django.contrib.staticfiles',
'bgpdata',
# 'tastypie',
)
API_LIMIT_PER_PAGE = 100
MIDDLEWARE_CLASSES = (
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
@ -76,6 +82,9 @@ USE_I18N = True @@ -76,6 +82,9 @@ USE_I18N = True
USE_L10N = True
USE_TZ = True
TEMPLATE_DIRS = (
'templates/',
)
# Static files (CSS, JavaScript, Images)

6
dnmapper/urls.py

@ -1,10 +1,14 @@ @@ -1,10 +1,14 @@
from django.conf.urls import patterns, include, url
from django.contrib import admin
from django.views.generic import RedirectView
import bgpdata.urls
urlpatterns = patterns('',
# Examples:
# url(r'^$', 'dnmapper.views.home', name='home'),
# url(r'^blog/', include('blog.urls')),
url(r'^$', RedirectView.as_view(url='/map/')),
url(r'^map/', include(bgpdata.urls)),
url(r'^admin/', include(admin.site.urls)),
url(r'^admin/', include(admin.site.urls)),
)

26
static/js/LICENSE

@ -0,0 +1,26 @@ @@ -0,0 +1,26 @@
Copyright (c) 2010-2015, Michael Bostock
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* The name Michael Bostock may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL MICHAEL BOSTOCK BE LIABLE FOR ANY DIRECT,
INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

9504
static/js/d3.js vendored

File diff suppressed because it is too large Load Diff

5
static/js/d3.min.js vendored

File diff suppressed because one or more lines are too long

10
templates/base.html

@ -0,0 +1,10 @@ @@ -0,0 +1,10 @@
<!doctype html5>
<html>
<head>
{% block head %}{% endblock %}
</head>
<body>
<h1>DarkMap</h1>
{% block body %}{% endblock %}
</body>
</html>

3
templates/templ.html

@ -0,0 +1,3 @@ @@ -0,0 +1,3 @@
{% extends "base.html" %}
{% block body %}
{% endblock %}
Loading…
Cancel
Save