Initial commit
This commit is contained in:
commit
12dfa7f3b4
|
|
@ -0,0 +1,5 @@
|
|||
*.pyc
|
||||
*.pyo
|
||||
.*.swp
|
||||
.*.swo
|
||||
db.sqlite3
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
from django.contrib import admin
|
||||
from bgpdata.models import ConfigHost, CrawlRun, CrawlLog, AS, BorderRouter, Announcement, Peering, BorderRouterPair
|
||||
|
||||
# Register your models here.
|
||||
admin.site.register(ConfigHost)
|
||||
admin.site.register(CrawlRun)
|
||||
admin.site.register(CrawlLog)
|
||||
admin.site.register(AS)
|
||||
admin.site.register(BorderRouter)
|
||||
admin.site.register(Announcement)
|
||||
admin.site.register(Peering)
|
||||
admin.site.register(BorderRouterPair)
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from django.db import models, migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='Announcement',
|
||||
fields=[
|
||||
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
|
||||
('ip', models.GenericIPAddressField()),
|
||||
('prefix', models.IntegerField()),
|
||||
('ASPath', models.CharField(max_length=512)),
|
||||
('nextHop', models.GenericIPAddressField()),
|
||||
],
|
||||
options={
|
||||
},
|
||||
bases=(models.Model,),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='AS',
|
||||
fields=[
|
||||
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
|
||||
('number', models.IntegerField()),
|
||||
],
|
||||
options={
|
||||
},
|
||||
bases=(models.Model,),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='BorderRouter',
|
||||
fields=[
|
||||
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
|
||||
('pingable', models.BooleanField(default=False)),
|
||||
('reachable', models.BooleanField(default=False)),
|
||||
('AS', models.ForeignKey(to='bgpdata.AS')),
|
||||
],
|
||||
options={
|
||||
},
|
||||
bases=(models.Model,),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='ConfigHost',
|
||||
fields=[
|
||||
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
|
||||
('number', models.IntegerField()),
|
||||
('ip', models.GenericIPAddressField()),
|
||||
('checkMethod', models.CharField(max_length=4, choices=[(b'CMK', b'Check MK'), (b'PLAIN', b'Plain')])),
|
||||
],
|
||||
options={
|
||||
},
|
||||
bases=(models.Model,),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='CrawlRun',
|
||||
fields=[
|
||||
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
|
||||
('startTime', models.DateTimeField()),
|
||||
('endTime', models.DateTimeField(blank=True)),
|
||||
],
|
||||
options={
|
||||
},
|
||||
bases=(models.Model,),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='Peering',
|
||||
fields=[
|
||||
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
|
||||
('origin', models.CharField(max_length=10, choices=[(b'path', b'BGP Path'), (b'direct', b'Direct Connection')])),
|
||||
('as1', models.ForeignKey(related_name='peering1', to='bgpdata.AS')),
|
||||
('as2', models.ForeignKey(related_name='peering2', to='bgpdata.AS')),
|
||||
('router1', models.ForeignKey(related_name='peering1', default=None, to='bgpdata.BorderRouter', null=True)),
|
||||
('router2', models.ForeignKey(related_name='peering2', default=None, to='bgpdata.BorderRouter', null=True)),
|
||||
],
|
||||
options={
|
||||
},
|
||||
bases=(models.Model,),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='as',
|
||||
name='crawl',
|
||||
field=models.ForeignKey(to='bgpdata.CrawlRun'),
|
||||
preserve_default=True,
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='announcement',
|
||||
name='originAS',
|
||||
field=models.ForeignKey(to='bgpdata.AS'),
|
||||
preserve_default=True,
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from django.db import models, migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('bgpdata', '0001_initial'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='confighost',
|
||||
name='name',
|
||||
field=models.CharField(default='<undef>', max_length=50),
|
||||
preserve_default=False,
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from django.db import models, migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('bgpdata', '0002_confighost_name'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='CrawlLog',
|
||||
fields=[
|
||||
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
|
||||
('logtime', models.DateTimeField(auto_now_add=True)),
|
||||
('severity', models.CharField(max_length=10, choices=[(b'INFO', b'info'), (b'ERROR', b'error'), (b'DEBUG', b'debug'), (b'WARN', b'warning')])),
|
||||
('message', models.TextField()),
|
||||
('crawl', models.ForeignKey(to='bgpdata.CrawlRun')),
|
||||
('host', models.ForeignKey(to='bgpdata.ConfigHost')),
|
||||
],
|
||||
options={
|
||||
},
|
||||
bases=(models.Model,),
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from django.db import models, migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('bgpdata', '0003_crawllog'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='crawlrun',
|
||||
name='endTime',
|
||||
field=models.DateTimeField(null=True, blank=True),
|
||||
preserve_default=True,
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from django.db import models, migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('bgpdata', '0004_auto_20150321_1607'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='crawllog',
|
||||
name='host',
|
||||
field=models.ForeignKey(to='bgpdata.ConfigHost', null=True),
|
||||
preserve_default=True,
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from django.db import models, migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('bgpdata', '0005_auto_20150321_1608'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='borderrouter',
|
||||
name='routerID',
|
||||
field=models.GenericIPAddressField(default=None),
|
||||
preserve_default=False,
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='crawllog',
|
||||
name='host',
|
||||
field=models.ForeignKey(blank=True, to='bgpdata.ConfigHost', null=True),
|
||||
preserve_default=True,
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from django.db import models, migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('bgpdata', '0006_auto_20150321_1845'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='BorderRouterPair',
|
||||
fields=[
|
||||
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
|
||||
('router1', models.ForeignKey(related_name='routerpair1', default=None, blank=True, to='bgpdata.BorderRouter', null=True)),
|
||||
('router2', models.ForeignKey(related_name='routerpair2', default=None, blank=True, to='bgpdata.BorderRouter', null=True)),
|
||||
],
|
||||
options={
|
||||
},
|
||||
bases=(models.Model,),
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='peering',
|
||||
name='router1',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='peering',
|
||||
name='router2',
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='announcement',
|
||||
name='router',
|
||||
field=models.ForeignKey(default=None, to='bgpdata.BorderRouter'),
|
||||
preserve_default=False,
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='peering',
|
||||
name='routers',
|
||||
field=models.ManyToManyField(default=None, to='bgpdata.BorderRouterPair', null=True, blank=True),
|
||||
preserve_default=True,
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from django.db import models, migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('bgpdata', '0007_auto_20150322_1828'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveField(
|
||||
model_name='peering',
|
||||
name='routers',
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='borderrouterpair',
|
||||
name='peering',
|
||||
field=models.ForeignKey(default=None, to='bgpdata.Peering'),
|
||||
preserve_default=False,
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,142 @@
|
|||
from django.db import models
|
||||
from django.db.models import Q
|
||||
|
||||
|
||||
# Create your models here.
|
||||
class ConfigHost(models.Model):
|
||||
CHECK_CHOICES = (
|
||||
('CMK', "Check MK"),
|
||||
('PLAIN', "Plain"),
|
||||
)
|
||||
|
||||
# asno, ip, check method,
|
||||
name = models.CharField(max_length=50)
|
||||
number = models.IntegerField()
|
||||
ip = models.GenericIPAddressField()
|
||||
checkMethod = models.CharField(max_length=4, choices=CHECK_CHOICES)
|
||||
|
||||
def __unicode__(self):
|
||||
return u"%s (%s / %s)" % (self.name, self.number, self.ip)
|
||||
|
||||
class CrawlRun(models.Model):
|
||||
# time start, time end,
|
||||
startTime = models.DateTimeField()
|
||||
endTime = models.DateTimeField(null=True, blank=True)
|
||||
|
||||
def __unicode__(self):
|
||||
return u"Run %d - %s to %s" % (self.pk, self.startTime, self.endTime if self.endTime else "?")
|
||||
|
||||
class CrawlLog(models.Model):
|
||||
INFO = 'INFO'
|
||||
ERROR = 'ERROR'
|
||||
DEBUG = 'DEBUG'
|
||||
WARN = 'WARN'
|
||||
SEVERITY = (
|
||||
(INFO, 'info'),
|
||||
(ERROR, 'error'),
|
||||
(DEBUG, 'debug'),
|
||||
(WARN, 'warning'),
|
||||
)
|
||||
|
||||
crawl = models.ForeignKey(CrawlRun)
|
||||
host = models.ForeignKey(ConfigHost, null=True, blank=True)
|
||||
logtime = models.DateTimeField(auto_now_add=True)
|
||||
severity = models.CharField(max_length=10, choices=SEVERITY)
|
||||
message = models.TextField()
|
||||
|
||||
@staticmethod
|
||||
def log(crawl, msg, severity=None, host=None):
|
||||
if not severity:
|
||||
severity = CrawlLog.ERROR
|
||||
|
||||
log = CrawlLog()
|
||||
log.crawl = crawl
|
||||
log.message = msg
|
||||
log.severity = severity
|
||||
log.host = host
|
||||
log.save()
|
||||
|
||||
def __unicode__(self):
|
||||
host = "host %s - " % self.host.name if self.host else ""
|
||||
return u"Log %s %s: %s%s" % (self.get_severity_display(), self.logtime, host, self.message)
|
||||
|
||||
class AS(models.Model):
|
||||
# asno
|
||||
crawl = models.ForeignKey(CrawlRun)
|
||||
number = models.IntegerField()
|
||||
|
||||
def __unicode__(self):
|
||||
return u"AS %s (crawl %d)" % (self.number, self.crawl.pk)
|
||||
|
||||
def getPeerings(self):
|
||||
return Peering.objects.filter(Q(as1=self)|Q(as2=self))
|
||||
|
||||
class BorderRouter(models.Model):
|
||||
# as id, ip, check method, pingable, reachable
|
||||
# unique: (crawl_id, asno, as id)
|
||||
AS = models.ForeignKey(AS)
|
||||
routerID = models.GenericIPAddressField()
|
||||
|
||||
pingable = models.BooleanField(default=False)
|
||||
reachable = models.BooleanField(default=False)
|
||||
|
||||
def __unicode__(self):
|
||||
p = "p" if self.pingable else "!p"
|
||||
r = "r" if self.reachable else "!r"
|
||||
return u"Router %s (AS %s, %s%s)" % (self.routerID, self.AS.number, p, r)
|
||||
|
||||
class Announcement(models.Model):
|
||||
router = models.ForeignKey(BorderRouter)
|
||||
|
||||
ip = models.GenericIPAddressField()
|
||||
prefix = models.IntegerField()
|
||||
|
||||
# NOTE: increase length for longer pathes (currently supports a length of ~85)
|
||||
ASPath = models.CharField(max_length=512)
|
||||
nextHop = models.GenericIPAddressField()
|
||||
originAS = models.ForeignKey(AS)
|
||||
|
||||
def __unicode__(self):
|
||||
return u"%s/%s via %s (crawl %s)" % (self.ip, self.prefix, self.ASPath, self.router.AS.crawl.pk)
|
||||
|
||||
class Peering(models.Model):
|
||||
DIRECT = 'direct'
|
||||
PATH = 'path'
|
||||
|
||||
ORIGIN = (
|
||||
(PATH, 'BGP Path'),
|
||||
(DIRECT, 'Direct Connection'),
|
||||
)
|
||||
|
||||
as1 = models.ForeignKey(AS, related_name='peering1')
|
||||
as2 = models.ForeignKey(AS, related_name='peering2')
|
||||
origin = models.CharField(max_length=10, choices=ORIGIN)
|
||||
|
||||
def __unicode__(self):
|
||||
return u"AS %s <--> AS %s (%s, crawl %s)" % (self.as1.number, self.as2.number, self.get_origin_display(), self.as1.crawl.pk)
|
||||
|
||||
def containsAS(self, AS):
|
||||
return AS in (self.as1, self.as2)
|
||||
|
||||
@staticmethod
|
||||
def getPeering(as1, as2):
|
||||
""" Find matching peering """
|
||||
try:
|
||||
return Peering.objects.get(as1=as1, as2=as2)
|
||||
except Peering.DoesNotExist:
|
||||
return Peering.objects.get(as1=as2, as2=as1)
|
||||
|
||||
class BorderRouterPair(models.Model):
|
||||
peering = models.ForeignKey(Peering)
|
||||
router1 = models.ForeignKey(BorderRouter, default=None, blank=True, null=True, related_name='routerpair1')
|
||||
router2 = models.ForeignKey(BorderRouter, default=None, blank=True, null=True, related_name='routerpair2')
|
||||
|
||||
def __unicode__(self):
|
||||
return u"%s <--> %s (crawl %d)" % (self.router1, self.router2, self.router1.AS.crawl.pk)
|
||||
|
||||
@staticmethod
|
||||
def getPairing(peering, router1, router2):
|
||||
try:
|
||||
return BorderRouterPair.objects.get(peering=peering, router1=router1, router2=router2)
|
||||
except BorderRouterPair.DoesNotExist:
|
||||
return BorderRouterPair.objects.get(peering=peering, router1=router2, router2=router1)
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
from django.test import TestCase
|
||||
|
||||
# Create your tests here.
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
from django.shortcuts import render
|
||||
|
||||
# Create your views here.
|
||||
|
|
@ -0,0 +1,155 @@
|
|||
#!/usr/bin/env python2
|
||||
from __future__ import print_function
|
||||
|
||||
# prepare environment
|
||||
import sys
|
||||
sys.path.append("..")
|
||||
import os
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dnmapper.settings")
|
||||
import django
|
||||
django.setup()
|
||||
|
||||
from django.utils import timezone
|
||||
from django.db.models import Q
|
||||
|
||||
from bgpdata.models import ConfigHost, CrawlRun, CrawlLog, AS, BorderRouter, Announcement, Peering, BorderRouterPair
|
||||
from routerparsers import getBGPData, RouterParserException
|
||||
|
||||
|
||||
def getOrCreateAS(crawl, number):
|
||||
currAS = None
|
||||
try:
|
||||
currAS = AS.objects.get(crawl=crawl, number=number)
|
||||
except AS.DoesNotExist:
|
||||
currAS = AS(crawl=crawl, number=number)
|
||||
currAS.save()
|
||||
|
||||
return currAS
|
||||
|
||||
def main():
|
||||
# 1. create crawl run
|
||||
crawl = CrawlRun()
|
||||
crawl.startTime = timezone.now()
|
||||
crawl.save()
|
||||
|
||||
CrawlLog.log(crawl, "Starting crawl run!", severity=CrawlLog.INFO)
|
||||
|
||||
# 2. get data from all hosts, put it in the database
|
||||
for host in ConfigHost.objects.all():
|
||||
data = None
|
||||
print(" -- Getting data for host %s" % host)
|
||||
try:
|
||||
if host.checkMethod == 'CMK':
|
||||
data = getBGPData(host.ip, host.number)
|
||||
else:
|
||||
CrawlLog.log(crawl, "Method %s is not currently supported, skipping host" % host.checkMethod, host=host, severity=CrawlLog.ERROR)
|
||||
continue
|
||||
except RouterParserException as e:
|
||||
msg = "Could not parse data for host: %s" % str(e)
|
||||
print("%s: %s" % (host, msg))
|
||||
CrawlLog.log(crawl, msg, host=host, severity=CrawlLog.ERROR)
|
||||
continue
|
||||
|
||||
print(" -- parsing...")
|
||||
|
||||
currASno = int(data["local_as"])
|
||||
currAS = getOrCreateAS(crawl, currASno)
|
||||
|
||||
currRouter = None
|
||||
try:
|
||||
currRouter = BorderRouter.objects.get(AS=currAS, routerID=data["local_id"])
|
||||
currRouter.pingable = True
|
||||
currRouter.reachable = True
|
||||
currRouter.save()
|
||||
except BorderRouter.DoesNotExist:
|
||||
currRouter = BorderRouter(AS=currAS, routerID=data["local_id"], pingable=True, reachable=True)
|
||||
currRouter.save()
|
||||
|
||||
print(" --> peers")
|
||||
for peer in data["peers"]:
|
||||
# peerings
|
||||
# data: BGP{state, neighbor_id, neighbor_as}, description
|
||||
|
||||
# a) find/create neighbor
|
||||
print(" ----> Peer:", int(peer["BGP"]["neighbor_as"]))
|
||||
neighAS = None
|
||||
try:
|
||||
neighAS = AS.objects.get(crawl=crawl, number=int(peer["BGP"]["neighbor_as"]))
|
||||
except AS.DoesNotExist:
|
||||
neighAS = AS(crawl=crawl, number=int(peer["BGP"]["neighbor_as"]))
|
||||
neighAS.save()
|
||||
|
||||
# b) find out if a peering already exists (maybe where we only need to add our router id?)
|
||||
peering = None
|
||||
try:
|
||||
peering = Peering.getPeering(currAS, neighAS)
|
||||
except Peering.DoesNotExist:
|
||||
peering = Peering(as1=currAS, as2=neighAS, origin=Peering.DIRECT)
|
||||
peering.save()
|
||||
|
||||
# c) look for router/peering pairs
|
||||
if peer["BGP"]["neighbor_id"]:
|
||||
try:
|
||||
neighRouter = BorderRouter.objects.get(AS=neighAS, routerID=peer["BGP"]["neighbor_id"])
|
||||
except BorderRouter.DoesNotExist:
|
||||
neighRouter = BorderRouter(AS=currAS, routerID=peer["BGP"]["neighbor_id"], pingable=False, reachable=False)
|
||||
neighRouter.save()
|
||||
try:
|
||||
BorderRouterPair.getPairing(peering, currRouter, neighRouter)
|
||||
except BorderRouterPair.DoesNotExist:
|
||||
pairs = BorderRouterPair.objects.filter(Q(peering=peering) & (Q(router1=neighRouter, router2=None)|Q(router1=None, router2=neighRouter)))
|
||||
if pairs.count() > 0:
|
||||
pair = pairs[0]
|
||||
if pair.router1 == None:
|
||||
pair.router1 = currRouter
|
||||
else:
|
||||
pair.router2 = currRouter
|
||||
pair.save()
|
||||
else:
|
||||
pair = BorderRouterPair(peering=peering, router1=currRouter, router2=neighRouter)
|
||||
pair.save()
|
||||
|
||||
print(" --> Announcements")
|
||||
if "routes" in data and data["routes"]:
|
||||
for route in data["routes"]:
|
||||
print(" ---->", route["prefix"])
|
||||
if "/" not in route["prefix"]:
|
||||
continue
|
||||
ip, prefix = route["prefix"].split("/")
|
||||
a = Announcement(router=currRouter, ip=ip, prefix=prefix,
|
||||
ASPath=" ".join(route["path"]), nextHop=route["nexthop"],
|
||||
originAS=currAS)
|
||||
a.save()
|
||||
else:
|
||||
print(" !! No routes found in host output")
|
||||
CrawlLog.log(crawl, "No routes found in host output (no bgp feed included?)", host=host, severity=CrawlLog.WARN)
|
||||
|
||||
# 3. calculate missing data
|
||||
print(" -- Adding extra data from announcements...")
|
||||
# 3.1. use announcement data to find hidden peerings
|
||||
for announcement in Announcement.objects.filter(router__AS__crawl=crawl):
|
||||
path = announcement.ASPath.split(" ")
|
||||
if len(path) > 1:
|
||||
firstASno = path.pop(0)
|
||||
firstAS = getOrCreateAS(crawl, firstASno)
|
||||
while len(path) > 0:
|
||||
secondASno = path.pop(0)
|
||||
secondAS = getOrCreateAS(crawl, secondASno)
|
||||
|
||||
try:
|
||||
Peering.getPeering(firstAS, secondAS)
|
||||
except Peering.DoesNotExist:
|
||||
peering = Peering(as1=firstAS, as2=secondAS, origin=Peering.PATH)
|
||||
peering.save()
|
||||
|
||||
firstAS = secondAS
|
||||
|
||||
# 4. end crawl run
|
||||
crawl.endTime = timezone.now()
|
||||
crawl.save()
|
||||
|
||||
print(" !! Done")
|
||||
CrawlLog.log(crawl, "Crawl completed", severity=CrawlLog.INFO)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -0,0 +1,340 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import re
|
||||
import socket
|
||||
|
||||
from collections import OrderedDict
|
||||
|
||||
class RouterParserException(Exception):
|
||||
pass
|
||||
|
||||
def err(msg):
|
||||
raise RouterParserException(msg)
|
||||
|
||||
|
||||
def getBGPData(ip, asno):
|
||||
rawData = getDataFromHost(ip)
|
||||
if not rawData:
|
||||
err("Could not get data from host (empty response)")
|
||||
|
||||
router = parseBGPData(rawData, asno)
|
||||
|
||||
router["ip"] = ip
|
||||
|
||||
return router
|
||||
|
||||
def getDataFromHost(ip):
|
||||
socket.setdefaulttimeout(5)
|
||||
x = socket.socket()
|
||||
x.connect((ip, 6556))
|
||||
f = x.makefile()
|
||||
data = f.read()
|
||||
x.close()
|
||||
|
||||
return data
|
||||
|
||||
def parseBGPData(raw, asno):
|
||||
d = re.search(r"(?:^|\n)<<<(quagga|bird)>>>\n(.*?)(?:$|<<<[^\n]+>>>)", raw, re.DOTALL)
|
||||
|
||||
if not d:
|
||||
err("Data not found in check mk output")
|
||||
|
||||
# mkify
|
||||
raw = d.group(2).split("\n")
|
||||
arr = filter(lambda _z: _z, map(lambda _y: filter(lambda _x: _x, re.split(r"\s+", _y)), raw))
|
||||
|
||||
# parse for bird/quagga
|
||||
result = None
|
||||
|
||||
if d.group(1) == "quagga":
|
||||
result = parseQuagga(arr, raw, asno)
|
||||
else:
|
||||
result = parseBird(arr, raw, asno)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def parseQuagga(data, raw, asno):
|
||||
status = _quaggaFindCommand(data, "show ip bgp sum")
|
||||
if status[0][0:3] != ['BGP', 'router', 'identifier']:
|
||||
print(status)
|
||||
err("Couldn't find router id in quagga output")
|
||||
|
||||
peers = _quaggaFindNeighbors(data)
|
||||
if asno and int(asno) != int(status[0][7]):
|
||||
err("AS number (%s) does not match as number from quagga (%s)" % (asno, status[0][7]))
|
||||
|
||||
routes = _quaggaFindRoutes(raw)
|
||||
|
||||
return {"local_id": status[0][3].strip(","), "local_as": int(status[0][7]), "peers": peers, "routes": routes}
|
||||
|
||||
def parseBird(data, raw, asno):
|
||||
status = _birdFindTable(data, "show status")
|
||||
if status[2][0] != "1011-Router":
|
||||
err("Couldn't find router id in bird output")
|
||||
peers = filter(lambda _x: _x["type"] == "BGP", _birdMakeProtocols(data))
|
||||
|
||||
if asno == None:
|
||||
err("Host is bird")
|
||||
# FIXME
|
||||
|
||||
routes = _birdFindRoutes(data)
|
||||
|
||||
return {"local_id": status[2][3], "local_as": int(asno), "peers": peers, "routes": routes}
|
||||
|
||||
def _birdFindTable(info, command):
|
||||
""" find command output of a bird command, e.g. "show bgp neighbors" """
|
||||
command = ["bird>"] + command.split(" ")
|
||||
commandInfo = []
|
||||
editNextLine = False
|
||||
for line in info:
|
||||
if not commandInfo:
|
||||
if line == command:
|
||||
commandInfo.append(line)
|
||||
editNextLine = True
|
||||
else:
|
||||
if editNextLine:
|
||||
editNextLine = False
|
||||
commandInfo.append(line[1:])
|
||||
elif line[0] == "bird>":
|
||||
return commandInfo
|
||||
else:
|
||||
commandInfo.append(line)
|
||||
return []
|
||||
|
||||
def _birdFindProtocols(info):
|
||||
""" return a list of tuples (protoname, protoinfo) """
|
||||
protocolTable = _birdFindTable(info, "show protocols all")
|
||||
protocols = OrderedDict()
|
||||
currProto = None
|
||||
for line in protocolTable[2:]:
|
||||
if line[0][0:4] == "1002":
|
||||
currProto = line[0][5:]
|
||||
protocols[currProto] = [[currProto] + line[1:]]
|
||||
elif currProto == None:
|
||||
err("No proto selected, couldn't parse line:", line)
|
||||
else:
|
||||
protocols[currProto].append(line)
|
||||
|
||||
return protocols
|
||||
|
||||
def _birdMakeProtocols(info):
|
||||
""" Parse birds show protocols all output """
|
||||
# proto: name, type, description, state (up/down?), up-since
|
||||
# routes imported, exported, preferred
|
||||
# also: routing stats (
|
||||
# bgp special stuff: state, neighbor (address, as, id) (id not available when down)
|
||||
# state (established, active)
|
||||
# if error, last error is avilable
|
||||
protocols = []
|
||||
for proto, data in _birdFindProtocols(info).iteritems():
|
||||
protoInfo = {
|
||||
"name": proto,
|
||||
"type": data[0][1],
|
||||
"table": data[0][2],
|
||||
"state": data[0][3],
|
||||
"last_change": data[0][4],
|
||||
"info": " ".join(data[0][5:]),
|
||||
"description": " ".join(data[1][2:]),
|
||||
"routes": {
|
||||
"imported": data[5][1],
|
||||
"exported": data[5][3],
|
||||
"preferred": data[5][5],
|
||||
}
|
||||
}
|
||||
if protoInfo["type"] == "BGP":
|
||||
found = False
|
||||
for n, line in enumerate(data):
|
||||
if line[0:2] == ["BGP", "state:"]:
|
||||
found = True
|
||||
protoInfo["BGP"] = {
|
||||
"state": data[n][2],
|
||||
"neighbor_address": data[n+1][2],
|
||||
"neighbor_as": int(data[n+2][2]),
|
||||
"neighbor_id": data[n+3][2] if data[n+3][0:2] == ["Neighbor", "ID:"] else None,
|
||||
"last_error": " ".join(data[n+3][2:]) if data[n+3][0:2] == ["Last", "error:"] else None,
|
||||
}
|
||||
|
||||
if not found:
|
||||
protoInfo["BGP"] = None
|
||||
|
||||
protocols.append(protoInfo)
|
||||
|
||||
return protocols
|
||||
|
||||
|
||||
def _birdFindRoutes(info):
|
||||
output = _birdFindTable(info, "show route all")
|
||||
if len(output) < 1:
|
||||
# no data found
|
||||
return None
|
||||
|
||||
def handleCandidate(routes, candidate):
|
||||
if candidate:
|
||||
# path, nexthop, network
|
||||
for key in ["path", "nexthop", "network", "iBGP"]:
|
||||
if key not in candidate:
|
||||
return
|
||||
route = {"prefix": candidate["network"], "nexthop": candidate["nexthop"], "path": candidate["path"], "iBGP": candidate["iBGP"]}
|
||||
routes.append(route)
|
||||
pass
|
||||
|
||||
routes = []
|
||||
candidate = None
|
||||
lastIP = None
|
||||
for line in output:
|
||||
if line[0].startswith("1007-"):
|
||||
# new route!
|
||||
handleCandidate(routes, candidate)
|
||||
if line[0] != "1007-":
|
||||
# line has a network, use it!
|
||||
lastIP = line[0][5:]
|
||||
candidate = {"network": lastIP, "iBGP": None}
|
||||
|
||||
elif candidate is not None:
|
||||
# search bgp attributes
|
||||
if line[0] == "1012-":
|
||||
pass
|
||||
k, v = line[1], line[2:]
|
||||
else:
|
||||
k, v = line[0], line[1:]
|
||||
|
||||
k = k.rstrip(":")
|
||||
if k == "BGP.next_hop":
|
||||
candidate["nexthop"] = v[0]
|
||||
elif k == "BGP.as_path":
|
||||
candidate["path"] = v
|
||||
|
||||
|
||||
handleCandidate(routes, candidate)
|
||||
|
||||
return routes
|
||||
|
||||
|
||||
def _quaggaFindCommand(info, cmd):
|
||||
# ['core-frunde#', 'show', 'ip', 'bgp', 'sum']
|
||||
# ['core-frunde#', 'show', 'ip', 'bgp', 'neighbors']
|
||||
output = []
|
||||
cmd = cmd.split(" ")
|
||||
prompt = None
|
||||
for line in info:
|
||||
if line[1:] == cmd:
|
||||
prompt = line[0]
|
||||
elif line[0] == prompt:
|
||||
# done
|
||||
return output
|
||||
elif prompt != None:
|
||||
output.append(line)
|
||||
|
||||
err("Could not find command '%s' in output" % " ".join(cmd))
|
||||
|
||||
def _quaggaFindNeighbors(info):
|
||||
#['BGP', 'neighbor', 'is', '10.50.1.2,', 'remote', 'AS', '65001,', 'local', 'AS', '65001,', 'internal', 'link']
|
||||
output = _quaggaFindCommand(info, "show ip bgp neighbors")
|
||||
start = ["BGP", "neighbor", "is"]
|
||||
|
||||
curr = None
|
||||
rawNeighbors = []
|
||||
for line in output:
|
||||
if line[0:3] == start:
|
||||
if curr:
|
||||
rawNeighbors.append(curr)
|
||||
curr = [line]
|
||||
elif curr:
|
||||
curr.append(line)
|
||||
else:
|
||||
err("Could not find start of neighbors")
|
||||
|
||||
if curr:
|
||||
rawNeighbors.append(curr)
|
||||
curr = None
|
||||
|
||||
neighbors = []
|
||||
neighborDict = OrderedDict()
|
||||
for raw in rawNeighbors:
|
||||
descrIdx = 1 if raw[1][0] == "Description:" else 0
|
||||
peerdict = {
|
||||
"neighbor_address": raw[0][3].rstrip(","),
|
||||
"neighbor_as": int(raw[0][6].rstrip(",")),
|
||||
"local_as": int(raw[0][9].rstrip(",")),
|
||||
"description": " ".join(raw[1][1:]) if descrIdx else "No description",
|
||||
"neighbor_id": raw[1+descrIdx][6].strip(","),
|
||||
"state": raw[2+descrIdx][3].strip(","),
|
||||
"routes": {
|
||||
"imported": 0,
|
||||
},
|
||||
"BGP": {
|
||||
"state": raw[2+descrIdx][3].strip(","),
|
||||
"neighbor_id": raw[1+descrIdx][6].strip(","),
|
||||
"neighbor_address": raw[0][3].rstrip(","),
|
||||
"neighbor_as": int(raw[0][6].rstrip(",")),
|
||||
"state": raw[2+descrIdx][3].strip(","),
|
||||
},
|
||||
}
|
||||
|
||||
for line in raw:
|
||||
if line[1:3] == ["accepted", "prefixes"]:
|
||||
# woooo
|
||||
peerdict["routes"]["imported"] = int(line[0])
|
||||
break
|
||||
|
||||
neighbors.append(peerdict)
|
||||
neighborDict[peerdict["neighbor_address"]] = peerdict
|
||||
|
||||
return neighbors
|
||||
|
||||
def _quaggaFindRoutes(raw):
|
||||
# from # show ip bgp to Total number of prefixes XX
|
||||
# BGP table version is 0, local router ID is 10.50.0.1
|
||||
# *> 10.3.14.0/27 10.75.0.22 0 65002 65112 i
|
||||
cmdre = re.compile(r"^([^\s#]+#) show ip bgp$")
|
||||
routere = re.compile(r"^(?P<status>.)(?P<status2>.)(?P<origin>.)(?P<network>[0-9./]+)?\s+(?P<nexthop>[0-9./]+)[\s0-9i]+$")
|
||||
|
||||
# find output
|
||||
output = []
|
||||
prompt = None
|
||||
for line in raw:
|
||||
if not prompt:
|
||||
m = cmdre.match(line)
|
||||
if m:
|
||||
prompt = m.group(1) + " "
|
||||
else:
|
||||
if line.startswith(prompt):
|
||||
break
|
||||
else:
|
||||
output.append(line)
|
||||
|
||||
if len(output) < 1:
|
||||
# no data found
|
||||
return None
|
||||
|
||||
routes = []
|
||||
foundTable = False
|
||||
lastIP = None
|
||||
for line in output:
|
||||
if not foundTable:
|
||||
if line.endswith("Metric LocPrf Weight Path"):
|
||||
foundTable = True
|
||||
else:
|
||||
if line != '':
|
||||
if line.startswith("Total number of prefixes"):
|
||||
break
|
||||
else:
|
||||
# parse one route line
|
||||
#print(line)
|
||||
m = routere.match(line)
|
||||
d = m.groupdict()
|
||||
if d["network"]:
|
||||
lastIP = d["network"]
|
||||
else:
|
||||
d["network"] = lastIP
|
||||
|
||||
# "parse" path (everything after 61 chars, but no i)
|
||||
path = filter(lambda _x: _x not in ('', 'i'), line[61:].split(" "))
|
||||
|
||||
route = {"prefix": d["network"], "nexthop": d["nexthop"], "path": path, "iBGP": d["origin"] == "i"}
|
||||
routes.append(route)
|
||||
|
||||
return routes
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
"""
|
||||
Django settings for dnmapper project.
|
||||
|
||||
For more information on this file, see
|
||||
https://docs.djangoproject.com/en/1.7/topics/settings/
|
||||
|
||||
For the full list of settings and their values, see
|
||||
https://docs.djangoproject.com/en/1.7/ref/settings/
|
||||
"""
|
||||
|
||||
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
|
||||
import os
|
||||
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
|
||||
|
||||
|
||||
# Quick-start development settings - unsuitable for production
|
||||
# See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/
|
||||
|
||||
# SECURITY WARNING: keep the secret key used in production secret!
|
||||
SECRET_KEY = 'crv*hfx2pkxvq1s!)dbz*hdu+r7u2$y4djf6_#6mm)shk9e!58'
|
||||
|
||||
# SECURITY WARNING: don't run with debug turned on in production!
|
||||
DEBUG = True
|
||||
|
||||
TEMPLATE_DEBUG = True
|
||||
|
||||
ALLOWED_HOSTS = []
|
||||
|
||||
|
||||
# Application definition
|
||||
|
||||
INSTALLED_APPS = (
|
||||
'django.contrib.admin',
|
||||
'django.contrib.auth',
|
||||
'django.contrib.contenttypes',
|
||||
'django.contrib.sessions',
|
||||
'django.contrib.messages',
|
||||
'django.contrib.staticfiles',
|
||||
'bgpdata',
|
||||
)
|
||||
|
||||
MIDDLEWARE_CLASSES = (
|
||||
'django.contrib.sessions.middleware.SessionMiddleware',
|
||||
'django.middleware.common.CommonMiddleware',
|
||||
'django.middleware.csrf.CsrfViewMiddleware',
|
||||
'django.contrib.auth.middleware.AuthenticationMiddleware',
|
||||
'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
|
||||
'django.contrib.messages.middleware.MessageMiddleware',
|
||||
'django.middleware.clickjacking.XFrameOptionsMiddleware',
|
||||
)
|
||||
|
||||
ROOT_URLCONF = 'dnmapper.urls'
|
||||
|
||||
WSGI_APPLICATION = 'dnmapper.wsgi.application'
|
||||
|
||||
|
||||
# Database
|
||||
# https://docs.djangoproject.com/en/1.7/ref/settings/#databases
|
||||
|
||||
DATABASES = {
|
||||
'default': {
|
||||
'ENGINE': 'django.db.backends.sqlite3',
|
||||
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
|
||||
}
|
||||
}
|
||||
|
||||
# Internationalization
|
||||
# https://docs.djangoproject.com/en/1.7/topics/i18n/
|
||||
|
||||
LANGUAGE_CODE = 'de-de'
|
||||
|
||||
TIME_ZONE = 'CET'
|
||||
|
||||
USE_I18N = True
|
||||
|
||||
USE_L10N = True
|
||||
|
||||
USE_TZ = True
|
||||
|
||||
|
||||
# Static files (CSS, JavaScript, Images)
|
||||
# https://docs.djangoproject.com/en/1.7/howto/static-files/
|
||||
|
||||
STATIC_URL = '/static/'
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
from django.conf.urls import patterns, include, url
|
||||
from django.contrib import admin
|
||||
|
||||
urlpatterns = patterns('',
|
||||
# Examples:
|
||||
# url(r'^$', 'dnmapper.views.home', name='home'),
|
||||
# url(r'^blog/', include('blog.urls')),
|
||||
|
||||
url(r'^admin/', include(admin.site.urls)),
|
||||
)
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
"""
|
||||
WSGI config for dnmapper project.
|
||||
|
||||
It exposes the WSGI callable as a module-level variable named ``application``.
|
||||
|
||||
For more information on this file, see
|
||||
https://docs.djangoproject.com/en/1.7/howto/deployment/wsgi/
|
||||
"""
|
||||
|
||||
import os
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dnmapper.settings")
|
||||
|
||||
from django.core.wsgi import get_wsgi_application
|
||||
application = get_wsgi_application()
|
||||
Loading…
Reference in New Issue