Migrate from tabs to spaces
Bow to the python best practices...
This commit is contained in:
parent
e98400e1b5
commit
525d8a724d
|
|
@ -7,34 +7,34 @@ from tastypie import fields
|
||||||
from bgpdata.models import AS, CrawlRun, Announcement, BorderRouter
|
from bgpdata.models import AS, CrawlRun, Announcement, BorderRouter
|
||||||
|
|
||||||
class ASResource(ModelResource):
|
class ASResource(ModelResource):
|
||||||
crawl = fields.ForeignKey("bgpdata.api.CrawlResource", "crawl")
|
crawl = fields.ForeignKey("bgpdata.api.CrawlResource", "crawl")
|
||||||
class Meta:
|
class Meta:
|
||||||
list_allowed_methods = ['get']
|
list_allowed_methods = ['get']
|
||||||
detail_allowed_methods = ['get']
|
detail_allowed_methods = ['get']
|
||||||
filtering = {'crawl': ALL_WITH_RELATIONS, 'number': ALL}
|
filtering = {'crawl': ALL_WITH_RELATIONS, 'number': ALL}
|
||||||
|
|
||||||
queryset = AS.objects.all()
|
queryset = AS.objects.all()
|
||||||
resource_name = "as"
|
resource_name = "as"
|
||||||
|
|
||||||
class CrawlResource(ModelResource):
|
class CrawlResource(ModelResource):
|
||||||
class Meta:
|
class Meta:
|
||||||
queryset = CrawlRun.objects.all()
|
queryset = CrawlRun.objects.all()
|
||||||
resource_name = "crawl"
|
resource_name = "crawl"
|
||||||
|
|
||||||
class BorderRouterResource(ModelResource):
|
class BorderRouterResource(ModelResource):
|
||||||
AS = fields.ForeignKey("bgpdata.api.ASResource", "AS")
|
AS = fields.ForeignKey("bgpdata.api.ASResource", "AS")
|
||||||
class Meta:
|
class Meta:
|
||||||
list_allowed_methods = ['get']
|
list_allowed_methods = ['get']
|
||||||
detail_allowed_methods = ['get']
|
detail_allowed_methods = ['get']
|
||||||
filtering = {'AS': ALL_WITH_RELATIONS}
|
filtering = {'AS': ALL_WITH_RELATIONS}
|
||||||
|
|
||||||
queryset = BorderRouter.objects.all()
|
queryset = BorderRouter.objects.all()
|
||||||
resource_name = "borderrouter"
|
resource_name = "borderrouter"
|
||||||
|
|
||||||
class AnnouncementResource(ModelResource):
|
class AnnouncementResource(ModelResource):
|
||||||
router = fields.ForeignKey("bgpdata.api.BorderRouterResource", "router")
|
router = fields.ForeignKey("bgpdata.api.BorderRouterResource", "router")
|
||||||
class Meta:
|
class Meta:
|
||||||
list_allowed_methods = ['get']
|
list_allowed_methods = ['get']
|
||||||
detail_allowed_methods = ['get']
|
detail_allowed_methods = ['get']
|
||||||
filtering = {'originAS': ALL_WITH_RELATIONS, 'crawlAS': ALL_WITH_RELATIONS, 'router': ALL_WITH_RELATIONS}
|
filtering = {'originAS': ALL_WITH_RELATIONS, 'crawlAS': ALL_WITH_RELATIONS, 'router': ALL_WITH_RELATIONS}
|
||||||
queryset = Announcement.objects.all()
|
queryset = Announcement.objects.all()
|
||||||
|
|
|
||||||
|
|
@ -8,185 +8,185 @@ from django.db.models import Q
|
||||||
|
|
||||||
# Create your models here.
|
# Create your models here.
|
||||||
class ConfigHost(models.Model):
|
class ConfigHost(models.Model):
|
||||||
CHECK_CHOICES = (
|
CHECK_CHOICES = (
|
||||||
('CMK', "Check MK"),
|
('CMK', "Check MK"),
|
||||||
('PLAIN', "Plain"),
|
('PLAIN', "Plain"),
|
||||||
)
|
)
|
||||||
|
|
||||||
# asno, ip, check method,
|
# asno, ip, check method,
|
||||||
name = models.CharField(max_length=50)
|
name = models.CharField(max_length=50)
|
||||||
number = models.IntegerField()
|
number = models.IntegerField()
|
||||||
ip = models.GenericIPAddressField()
|
ip = models.GenericIPAddressField()
|
||||||
checkMethod = models.CharField(max_length=4, choices=CHECK_CHOICES)
|
checkMethod = models.CharField(max_length=4, choices=CHECK_CHOICES)
|
||||||
|
|
||||||
def __unicode__(self):
|
def __unicode__(self):
|
||||||
return u"%s (%s / %s)" % (self.name, self.number, self.ip)
|
return u"%s (%s / %s)" % (self.name, self.number, self.ip)
|
||||||
|
|
||||||
class CrawlRun(models.Model):
|
class CrawlRun(models.Model):
|
||||||
# time start, time end,
|
# time start, time end,
|
||||||
startTime = models.DateTimeField()
|
startTime = models.DateTimeField()
|
||||||
endTime = models.DateTimeField(null=True, blank=True)
|
endTime = models.DateTimeField(null=True, blank=True)
|
||||||
|
|
||||||
hostsCrawled = models.ManyToManyField(ConfigHost, null=True, blank=True)
|
hostsCrawled = models.ManyToManyField(ConfigHost, null=True, blank=True)
|
||||||
|
|
||||||
asCount = models.IntegerField(default=0)
|
asCount = models.IntegerField(default=0)
|
||||||
asOnlineCount = models.IntegerField(default=0)
|
asOnlineCount = models.IntegerField(default=0)
|
||||||
asOfflineCount = models.IntegerField(default=0)
|
asOfflineCount = models.IntegerField(default=0)
|
||||||
peeringCount = models.IntegerField(default=0)
|
peeringCount = models.IntegerField(default=0)
|
||||||
|
|
||||||
def __unicode__(self):
|
def __unicode__(self):
|
||||||
return u"Run %d - %s to %s" % (self.pk, self.startTime, self.endTime if self.endTime else "?")
|
return u"Run %d - %s to %s" % (self.pk, self.startTime, self.endTime if self.endTime else "?")
|
||||||
|
|
||||||
def countAS(self):
|
def countAS(self):
|
||||||
return self.asCount
|
return self.asCount
|
||||||
def countASOnline(self):
|
def countASOnline(self):
|
||||||
return self.asOnlineCount
|
return self.asOnlineCount
|
||||||
def countASOffline(self):
|
def countASOffline(self):
|
||||||
return self.asOfflineCount
|
return self.asOfflineCount
|
||||||
#return self.as_set.filter(online=False).count()
|
#return self.as_set.filter(online=False).count()
|
||||||
def countPeerings(self):
|
def countPeerings(self):
|
||||||
return self.peeringCount
|
return self.peeringCount
|
||||||
#return Peering.objects.filter(Q(as1__crawl=self)|Q(as2__crawl=self)).count()
|
#return Peering.objects.filter(Q(as1__crawl=self)|Q(as2__crawl=self)).count()
|
||||||
|
|
||||||
class CrawlLog(models.Model):
|
class CrawlLog(models.Model):
|
||||||
INFO = 'INFO'
|
INFO = 'INFO'
|
||||||
ERROR = 'ERROR'
|
ERROR = 'ERROR'
|
||||||
DEBUG = 'DEBUG'
|
DEBUG = 'DEBUG'
|
||||||
WARN = 'WARN'
|
WARN = 'WARN'
|
||||||
SEVERITY = (
|
SEVERITY = (
|
||||||
(INFO, 'info'),
|
(INFO, 'info'),
|
||||||
(ERROR, 'error'),
|
(ERROR, 'error'),
|
||||||
(DEBUG, 'debug'),
|
(DEBUG, 'debug'),
|
||||||
(WARN, 'warning'),
|
(WARN, 'warning'),
|
||||||
)
|
)
|
||||||
|
|
||||||
crawl = models.ForeignKey(CrawlRun)
|
crawl = models.ForeignKey(CrawlRun)
|
||||||
host = models.ForeignKey(ConfigHost, null=True, blank=True, on_delete=models.SET_NULL)
|
host = models.ForeignKey(ConfigHost, null=True, blank=True, on_delete=models.SET_NULL)
|
||||||
logtime = models.DateTimeField(auto_now_add=True)
|
logtime = models.DateTimeField(auto_now_add=True)
|
||||||
severity = models.CharField(max_length=10, choices=SEVERITY)
|
severity = models.CharField(max_length=10, choices=SEVERITY)
|
||||||
message = models.TextField()
|
message = models.TextField()
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def log(crawl, msg, severity=None, host=None):
|
def log(crawl, msg, severity=None, host=None):
|
||||||
if not severity:
|
if not severity:
|
||||||
severity = CrawlLog.ERROR
|
severity = CrawlLog.ERROR
|
||||||
|
|
||||||
log = CrawlLog()
|
log = CrawlLog()
|
||||||
log.crawl = crawl
|
log.crawl = crawl
|
||||||
log.message = msg
|
log.message = msg
|
||||||
log.severity = severity
|
log.severity = severity
|
||||||
log.host = host
|
log.host = host
|
||||||
log.save()
|
log.save()
|
||||||
|
|
||||||
def __unicode__(self):
|
def __unicode__(self):
|
||||||
host = "host %s - " % self.host.name if self.host else ""
|
host = "host %s - " % self.host.name if self.host else ""
|
||||||
return u"Log %s %s: %s%s" % (self.get_severity_display(), self.logtime, host, self.message)
|
return u"Log %s %s: %s%s" % (self.get_severity_display(), self.logtime, host, self.message)
|
||||||
|
|
||||||
class AS(models.Model):
|
class AS(models.Model):
|
||||||
# asno
|
# asno
|
||||||
crawl = models.ForeignKey(CrawlRun)
|
crawl = models.ForeignKey(CrawlRun)
|
||||||
number = models.IntegerField(db_index=True)
|
number = models.IntegerField(db_index=True)
|
||||||
|
|
||||||
directlyCrawled = models.BooleanField(default=False)
|
directlyCrawled = models.BooleanField(default=False)
|
||||||
online = models.BooleanField(default=True, db_index=True)
|
online = models.BooleanField(default=True, db_index=True)
|
||||||
lastSeen = models.ForeignKey(CrawlRun, blank=True, null=True, default=None, related_name='as_lastseen')
|
lastSeen = models.ForeignKey(CrawlRun, blank=True, null=True, default=None, related_name='as_lastseen')
|
||||||
|
|
||||||
class Meta:
|
class Meta:
|
||||||
unique_together = (('crawl', 'number'),)
|
unique_together = (('crawl', 'number'),)
|
||||||
index_together = (
|
index_together = (
|
||||||
('crawl', 'number'),
|
('crawl', 'number'),
|
||||||
)
|
)
|
||||||
|
|
||||||
def __unicode__(self):
|
def __unicode__(self):
|
||||||
return u"AS %s (crawl %d)" % (self.number, self.crawl.pk)
|
return u"AS %s (crawl %d)" % (self.number, self.crawl.pk)
|
||||||
|
|
||||||
def setOnline(self):
|
def setOnline(self):
|
||||||
if not self.online:
|
if not self.online:
|
||||||
self.online = True
|
self.online = True
|
||||||
self.lastSeen = None
|
self.lastSeen = None
|
||||||
self.save()
|
self.save()
|
||||||
|
|
||||||
def getPeerings(self):
|
def getPeerings(self):
|
||||||
return Peering.objects.filter(Q(as1=self)|Q(as2=self))
|
return Peering.objects.filter(Q(as1=self)|Q(as2=self))
|
||||||
|
|
||||||
def getAnnouncedPrefixes(self):
|
def getAnnouncedPrefixes(self):
|
||||||
return list(set(map(lambda _x: "%(ip)s/%(prefix)s" % _x, self.announcement_set.all().values('ip', 'prefix'))))
|
return list(set(map(lambda _x: "%(ip)s/%(prefix)s" % _x, self.announcement_set.all().values('ip', 'prefix'))))
|
||||||
|
|
||||||
def formatLastSeen(self):
|
def formatLastSeen(self):
|
||||||
if self.lastSeen:
|
if self.lastSeen:
|
||||||
return self.lastSeen.startTime.strftime("%d.%m.%Y %H:%I")
|
return self.lastSeen.startTime.strftime("%d.%m.%Y %H:%I")
|
||||||
|
|
||||||
class BorderRouter(models.Model):
|
class BorderRouter(models.Model):
|
||||||
# as id, ip, check method, pingable, reachable
|
# as id, ip, check method, pingable, reachable
|
||||||
# unique: (crawl_id, asno, as id)
|
# unique: (crawl_id, asno, as id)
|
||||||
AS = models.ForeignKey(AS)
|
AS = models.ForeignKey(AS)
|
||||||
routerID = models.GenericIPAddressField()
|
routerID = models.GenericIPAddressField()
|
||||||
|
|
||||||
pingable = models.BooleanField(default=False)
|
pingable = models.BooleanField(default=False)
|
||||||
reachable = models.BooleanField(default=False)
|
reachable = models.BooleanField(default=False)
|
||||||
|
|
||||||
def __unicode__(self):
|
def __unicode__(self):
|
||||||
p = "p" if self.pingable else "!p"
|
p = "p" if self.pingable else "!p"
|
||||||
r = "r" if self.reachable else "!r"
|
r = "r" if self.reachable else "!r"
|
||||||
return u"Router %s (AS %s, %s%s)" % (self.routerID, self.AS.number, p, r)
|
return u"Router %s (AS %s, %s%s)" % (self.routerID, self.AS.number, p, r)
|
||||||
|
|
||||||
class Announcement(models.Model):
|
class Announcement(models.Model):
|
||||||
router = models.ForeignKey(BorderRouter)
|
router = models.ForeignKey(BorderRouter)
|
||||||
|
|
||||||
ip = models.GenericIPAddressField()
|
ip = models.GenericIPAddressField()
|
||||||
prefix = models.IntegerField()
|
prefix = models.IntegerField()
|
||||||
|
|
||||||
# NOTE: increase length for longer pathes (currently supports a length of ~85)
|
# NOTE: increase length for longer pathes (currently supports a length of ~85)
|
||||||
ASPath = models.CharField(max_length=512)
|
ASPath = models.CharField(max_length=512)
|
||||||
nextHop = models.GenericIPAddressField()
|
nextHop = models.GenericIPAddressField()
|
||||||
originAS = models.ForeignKey(AS, null=True)
|
originAS = models.ForeignKey(AS, null=True)
|
||||||
crawlAS = models.ForeignKey(AS, related_name='crawl_as', null=True)
|
crawlAS = models.ForeignKey(AS, related_name='crawl_as', null=True)
|
||||||
|
|
||||||
def __unicode__(self):
|
def __unicode__(self):
|
||||||
return u"%s/%s via %s (crawl %s)" % (self.ip, self.prefix, self.ASPath, self.router.AS.crawl.pk)
|
return u"%s/%s via %s (crawl %s)" % (self.ip, self.prefix, self.ASPath, self.router.AS.crawl.pk)
|
||||||
|
|
||||||
class Peering(models.Model):
|
class Peering(models.Model):
|
||||||
DIRECT = 'direct'
|
DIRECT = 'direct'
|
||||||
PATH = 'path'
|
PATH = 'path'
|
||||||
|
|
||||||
ORIGIN = (
|
ORIGIN = (
|
||||||
(PATH, 'BGP Path'),
|
(PATH, 'BGP Path'),
|
||||||
(DIRECT, 'Direct Connection'),
|
(DIRECT, 'Direct Connection'),
|
||||||
)
|
)
|
||||||
|
|
||||||
index_together = (
|
index_together = (
|
||||||
('as1', 'as2'),
|
('as1', 'as2'),
|
||||||
)
|
)
|
||||||
|
|
||||||
as1 = models.ForeignKey(AS, related_name='peering1')
|
as1 = models.ForeignKey(AS, related_name='peering1')
|
||||||
as2 = models.ForeignKey(AS, related_name='peering2')
|
as2 = models.ForeignKey(AS, related_name='peering2')
|
||||||
origin = models.CharField(max_length=10, choices=ORIGIN)
|
origin = models.CharField(max_length=10, choices=ORIGIN)
|
||||||
|
|
||||||
def __unicode__(self):
|
def __unicode__(self):
|
||||||
return u"AS %s <--> AS %s (%s, crawl %s)" % (self.as1.number, self.as2.number, self.get_origin_display(), self.as1.crawl.pk)
|
return u"AS %s <--> AS %s (%s, crawl %s)" % (self.as1.number, self.as2.number, self.get_origin_display(), self.as1.crawl.pk)
|
||||||
|
|
||||||
def containsAS(self, AS):
|
def containsAS(self, AS):
|
||||||
return AS in (self.as1, self.as2)
|
return AS in (self.as1, self.as2)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def getPeering(as1, as2):
|
def getPeering(as1, as2):
|
||||||
""" Find matching peering """
|
""" Find matching peering """
|
||||||
try:
|
try:
|
||||||
return Peering.objects.get(as1=as1, as2=as2)
|
return Peering.objects.get(as1=as1, as2=as2)
|
||||||
except Peering.DoesNotExist:
|
except Peering.DoesNotExist:
|
||||||
return Peering.objects.get(as1=as2, as2=as1)
|
return Peering.objects.get(as1=as2, as2=as1)
|
||||||
|
|
||||||
class BorderRouterPair(models.Model):
|
class BorderRouterPair(models.Model):
|
||||||
peering = models.ForeignKey(Peering)
|
peering = models.ForeignKey(Peering)
|
||||||
router1 = models.ForeignKey(BorderRouter, default=None, blank=True, null=True, related_name='routerpair1')
|
router1 = models.ForeignKey(BorderRouter, default=None, blank=True, null=True, related_name='routerpair1')
|
||||||
router2 = models.ForeignKey(BorderRouter, default=None, blank=True, null=True, related_name='routerpair2')
|
router2 = models.ForeignKey(BorderRouter, default=None, blank=True, null=True, related_name='routerpair2')
|
||||||
|
|
||||||
def __unicode__(self):
|
def __unicode__(self):
|
||||||
return u"%s <--> %s (crawl %d)" % (self.router1, self.router2, self.router1.AS.crawl.pk)
|
return u"%s <--> %s (crawl %d)" % (self.router1, self.router2, self.router1.AS.crawl.pk)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def getPairing(peering, router1, router2):
|
def getPairing(peering, router1, router2):
|
||||||
try:
|
try:
|
||||||
return BorderRouterPair.objects.get(peering=peering, router1=router1, router2=router2)
|
return BorderRouterPair.objects.get(peering=peering, router1=router1, router2=router2)
|
||||||
except BorderRouterPair.DoesNotExist:
|
except BorderRouterPair.DoesNotExist:
|
||||||
return BorderRouterPair.objects.get(peering=peering, router1=router2, router2=router1)
|
return BorderRouterPair.objects.get(peering=peering, router1=router2, router2=router1)
|
||||||
|
|
|
||||||
|
|
@ -13,14 +13,14 @@ borderRouterResource = BorderRouterResource()
|
||||||
announcementResource = AnnouncementResource()
|
announcementResource = AnnouncementResource()
|
||||||
|
|
||||||
urlpatterns = (
|
urlpatterns = (
|
||||||
url(r'^$', bgpdata_views.overview),
|
url(r'^$', bgpdata_views.overview),
|
||||||
url(r'^([0-9]+)/$', bgpdata_views.showMap),
|
url(r'^([0-9]+)/$', bgpdata_views.showMap),
|
||||||
|
|
||||||
#url(r'^api/crawl/(?P<crawlID>\d+)/asses/$', 'bgpdata.api.asses'),
|
#url(r'^api/crawl/(?P<crawlID>\d+)/asses/$', 'bgpdata.api.asses'),
|
||||||
#(r'^api/', include(asResource.urls)),
|
#(r'^api/', include(asResource.urls)),
|
||||||
url(r'^api/', include(asResource.urls)),
|
url(r'^api/', include(asResource.urls)),
|
||||||
url(r'^api/', include(crawlResource.urls)),
|
url(r'^api/', include(crawlResource.urls)),
|
||||||
url(r'^api/', include(borderRouterResource.urls)),
|
url(r'^api/', include(borderRouterResource.urls)),
|
||||||
url(r'^api/', include(announcementResource.urls)),
|
url(r'^api/', include(announcementResource.urls)),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -7,18 +7,18 @@ from bgpdata.models import CrawlRun, AS, Peering
|
||||||
from django.core.paginator import Paginator
|
from django.core.paginator import Paginator
|
||||||
|
|
||||||
def overview(request):
|
def overview(request):
|
||||||
crawls = CrawlRun.objects.order_by("-startTime")
|
crawls = CrawlRun.objects.order_by("-startTime")
|
||||||
crawlsPage = Paginator(crawls, 200)
|
crawlsPage = Paginator(crawls, 200)
|
||||||
return render(request, 'bgpdata/overview.html', {"crawls": crawlsPage.page(1)})
|
return render(request, 'bgpdata/overview.html', {"crawls": crawlsPage.page(1)})
|
||||||
|
|
||||||
def showMap(request, crawlId):
|
def showMap(request, crawlId):
|
||||||
crawl = None
|
crawl = None
|
||||||
try:
|
try:
|
||||||
crawl = CrawlRun.objects.get(id=crawlId)
|
crawl = CrawlRun.objects.get(id=crawlId)
|
||||||
except CrawlRun.DoesNotExist:
|
except CrawlRun.DoesNotExist:
|
||||||
return render(request, "bgpdata/no-map-found.html", {"crawl_id": crawlId})
|
return render(request, "bgpdata/no-map-found.html", {"crawl_id": crawlId})
|
||||||
|
|
||||||
ASses = AS.objects.filter(crawl=crawl)
|
ASses = AS.objects.filter(crawl=crawl)
|
||||||
peerings = Peering.objects.filter(as1__crawl=crawl)
|
peerings = Peering.objects.filter(as1__crawl=crawl)
|
||||||
|
|
||||||
return render(request, 'bgpdata/map.html', {"crawl": crawl, 'ASses': ASses, 'peerings': peerings})
|
return render(request, 'bgpdata/map.html', {"crawl": crawl, 'ASses': ASses, 'peerings': peerings})
|
||||||
|
|
|
||||||
324
bin/crawl.py
324
bin/crawl.py
|
|
@ -26,193 +26,193 @@ from routerparsers import getBGPData, RouterParserException
|
||||||
|
|
||||||
|
|
||||||
def getOrCreateAS(crawl, number, online=True):
|
def getOrCreateAS(crawl, number, online=True):
|
||||||
currAS = None
|
currAS = None
|
||||||
try:
|
try:
|
||||||
currAS = AS.objects.get(crawl=crawl, number=number)
|
currAS = AS.objects.get(crawl=crawl, number=number)
|
||||||
if online:
|
if online:
|
||||||
currAS.setOnline()
|
currAS.setOnline()
|
||||||
except AS.DoesNotExist:
|
except AS.DoesNotExist:
|
||||||
currAS = AS(crawl=crawl, number=number, online=online)
|
currAS = AS(crawl=crawl, number=number, online=online)
|
||||||
currAS.save()
|
currAS.save()
|
||||||
|
|
||||||
return currAS
|
return currAS
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
# 1. create crawl run
|
# 1. create crawl run
|
||||||
crawl = CrawlRun()
|
crawl = CrawlRun()
|
||||||
crawl.startTime = timezone.now()
|
crawl.startTime = timezone.now()
|
||||||
crawl.save()
|
crawl.save()
|
||||||
|
|
||||||
CrawlLog.log(crawl, "Starting crawl run!", severity=CrawlLog.INFO)
|
CrawlLog.log(crawl, "Starting crawl run!", severity=CrawlLog.INFO)
|
||||||
|
|
||||||
# 2. get data from all hosts, put it in the database
|
# 2. get data from all hosts, put it in the database
|
||||||
for host in ConfigHost.objects.all():
|
for host in ConfigHost.objects.all():
|
||||||
crawl.hostsCrawled.add(host)
|
crawl.hostsCrawled.add(host)
|
||||||
data = None
|
data = None
|
||||||
print(" -- Getting data for host %s" % host)
|
print(" -- Getting data for host %s" % host)
|
||||||
try:
|
try:
|
||||||
if host.checkMethod == 'CMK':
|
if host.checkMethod == 'CMK':
|
||||||
data = getBGPData(host.ip, host.number)
|
data = getBGPData(host.ip, host.number)
|
||||||
else:
|
else:
|
||||||
CrawlLog.log(crawl, "Method %s is not currently supported, skipping host" % host.checkMethod, host=host, severity=CrawlLog.ERROR)
|
CrawlLog.log(crawl, "Method %s is not currently supported, skipping host" % host.checkMethod, host=host, severity=CrawlLog.ERROR)
|
||||||
continue
|
continue
|
||||||
except RouterParserException as e:
|
except RouterParserException as e:
|
||||||
msg = "Could not parse data for host: %s" % str(e)
|
msg = "Could not parse data for host: %s" % str(e)
|
||||||
print("%s: %s" % (host, msg))
|
print("%s: %s" % (host, msg))
|
||||||
CrawlLog.log(crawl, msg, host=host, severity=CrawlLog.ERROR)
|
CrawlLog.log(crawl, msg, host=host, severity=CrawlLog.ERROR)
|
||||||
continue
|
continue
|
||||||
except socket.error as e:
|
except socket.error as e:
|
||||||
msg = "Could not reach host: %s" % (e,)
|
msg = "Could not reach host: %s" % (e,)
|
||||||
print("%s: %s" % (host, msg))
|
print("%s: %s" % (host, msg))
|
||||||
CrawlLog.log(crawl, msg, host=host, severity=CrawlLog.ERROR)
|
CrawlLog.log(crawl, msg, host=host, severity=CrawlLog.ERROR)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
print(" -- parsing...")
|
print(" -- parsing...")
|
||||||
|
|
||||||
currASno = int(data["local_as"])
|
currASno = int(data["local_as"])
|
||||||
currAS = getOrCreateAS(crawl, currASno)
|
currAS = getOrCreateAS(crawl, currASno)
|
||||||
|
|
||||||
currAS.directlyCrawled = True
|
currAS.directlyCrawled = True
|
||||||
currAS.save()
|
currAS.save()
|
||||||
|
|
||||||
currRouter = None
|
currRouter = None
|
||||||
try:
|
try:
|
||||||
currRouter = BorderRouter.objects.get(AS=currAS, routerID=data["local_id"])
|
currRouter = BorderRouter.objects.get(AS=currAS, routerID=data["local_id"])
|
||||||
currRouter.pingable = True
|
currRouter.pingable = True
|
||||||
currRouter.reachable = True
|
currRouter.reachable = True
|
||||||
currRouter.save()
|
currRouter.save()
|
||||||
except BorderRouter.DoesNotExist:
|
except BorderRouter.DoesNotExist:
|
||||||
currRouter = BorderRouter(AS=currAS, routerID=data["local_id"], pingable=True, reachable=True)
|
currRouter = BorderRouter(AS=currAS, routerID=data["local_id"], pingable=True, reachable=True)
|
||||||
currRouter.save()
|
currRouter.save()
|
||||||
|
|
||||||
print(" --> peers")
|
print(" --> peers")
|
||||||
for peer in data["peers"]:
|
for peer in data["peers"]:
|
||||||
# peerings
|
# peerings
|
||||||
# data: BGP{state, neighbor_id, neighbor_as}, description
|
# data: BGP{state, neighbor_id, neighbor_as}, description
|
||||||
|
|
||||||
# a) find/create neighbor
|
# a) find/create neighbor
|
||||||
print(" ----> Peer:", int(peer["BGP"]["neighbor_as"]))
|
print(" ----> Peer:", int(peer["BGP"]["neighbor_as"]))
|
||||||
neighAS = getOrCreateAS(crawl, int(peer["BGP"]["neighbor_as"]), online=peer["BGP"]["online"])
|
neighAS = getOrCreateAS(crawl, int(peer["BGP"]["neighbor_as"]), online=peer["BGP"]["online"])
|
||||||
|
|
||||||
# b) find out if a peering already exists (maybe where we only need to add our router id?)
|
# b) find out if a peering already exists (maybe where we only need to add our router id?)
|
||||||
peering = None
|
peering = None
|
||||||
try:
|
try:
|
||||||
peering = Peering.getPeering(currAS, neighAS)
|
peering = Peering.getPeering(currAS, neighAS)
|
||||||
except Peering.DoesNotExist:
|
except Peering.DoesNotExist:
|
||||||
peering = Peering(as1=currAS, as2=neighAS, origin=Peering.DIRECT)
|
peering = Peering(as1=currAS, as2=neighAS, origin=Peering.DIRECT)
|
||||||
peering.save()
|
peering.save()
|
||||||
|
|
||||||
# c) look for router/peering pairs
|
# c) look for router/peering pairs
|
||||||
if peer["BGP"]["neighbor_id"]:
|
if peer["BGP"]["neighbor_id"]:
|
||||||
try:
|
try:
|
||||||
neighRouter = BorderRouter.objects.get(AS=neighAS, routerID=peer["BGP"]["neighbor_id"])
|
neighRouter = BorderRouter.objects.get(AS=neighAS, routerID=peer["BGP"]["neighbor_id"])
|
||||||
except BorderRouter.DoesNotExist:
|
except BorderRouter.DoesNotExist:
|
||||||
neighRouter = BorderRouter(AS=neighAS, routerID=peer["BGP"]["neighbor_id"], pingable=False, reachable=False)
|
neighRouter = BorderRouter(AS=neighAS, routerID=peer["BGP"]["neighbor_id"], pingable=False, reachable=False)
|
||||||
neighRouter.save()
|
neighRouter.save()
|
||||||
try:
|
try:
|
||||||
BorderRouterPair.getPairing(peering, currRouter, neighRouter)
|
BorderRouterPair.getPairing(peering, currRouter, neighRouter)
|
||||||
except BorderRouterPair.DoesNotExist:
|
except BorderRouterPair.DoesNotExist:
|
||||||
pairs = BorderRouterPair.objects.filter(Q(peering=peering) & (Q(router1=neighRouter, router2=None)|Q(router1=None, router2=neighRouter)))
|
pairs = BorderRouterPair.objects.filter(Q(peering=peering) & (Q(router1=neighRouter, router2=None)|Q(router1=None, router2=neighRouter)))
|
||||||
if pairs.count() > 0:
|
if pairs.count() > 0:
|
||||||
pair = pairs[0]
|
pair = pairs[0]
|
||||||
if pair.router1 == None:
|
if pair.router1 == None:
|
||||||
pair.router1 = currRouter
|
pair.router1 = currRouter
|
||||||
else:
|
else:
|
||||||
pair.router2 = currRouter
|
pair.router2 = currRouter
|
||||||
pair.save()
|
pair.save()
|
||||||
else:
|
else:
|
||||||
pair = BorderRouterPair(peering=peering, router1=currRouter, router2=neighRouter)
|
pair = BorderRouterPair(peering=peering, router1=currRouter, router2=neighRouter)
|
||||||
pair.save()
|
pair.save()
|
||||||
|
|
||||||
print(" --> Announcements")
|
print(" --> Announcements")
|
||||||
if "routes" in data and data["routes"]:
|
if "routes" in data and data["routes"]:
|
||||||
for route in data["routes"]:
|
for route in data["routes"]:
|
||||||
print(" ---->", route["prefix"])
|
print(" ---->", route["prefix"])
|
||||||
if "/" not in route["prefix"]:
|
if "/" not in route["prefix"]:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
crawlAS = currAS
|
crawlAS = currAS
|
||||||
if len(route["path"]) > 0:
|
if len(route["path"]) > 0:
|
||||||
crawlAS = getOrCreateAS(crawl, route["path"][0])
|
crawlAS = getOrCreateAS(crawl, route["path"][0])
|
||||||
originAS = getOrCreateAS(crawl, route["path"][-1])
|
originAS = getOrCreateAS(crawl, route["path"][-1])
|
||||||
ip, prefix = route["prefix"].split("/")
|
ip, prefix = route["prefix"].split("/")
|
||||||
a = Announcement(router=currRouter, ip=ip, prefix=prefix,
|
a = Announcement(router=currRouter, ip=ip, prefix=prefix,
|
||||||
ASPath=" ".join(route["path"]), nextHop=route["nexthop"],
|
ASPath=" ".join(route["path"]), nextHop=route["nexthop"],
|
||||||
crawlAS=crawlAS, originAS=originAS)
|
crawlAS=crawlAS, originAS=originAS)
|
||||||
a.save()
|
a.save()
|
||||||
else:
|
else:
|
||||||
print(" !! No routes found in host output")
|
print(" !! No routes found in host output")
|
||||||
CrawlLog.log(crawl, "No routes found in host output (no bgp feed included?)", host=host, severity=CrawlLog.WARN)
|
CrawlLog.log(crawl, "No routes found in host output (no bgp feed included?)", host=host, severity=CrawlLog.WARN)
|
||||||
|
|
||||||
# 3. calculate missing data
|
# 3. calculate missing data
|
||||||
print(" -- Adding extra data from announcements...")
|
print(" -- Adding extra data from announcements...")
|
||||||
# 3.1. use announcement data to find hidden peerings
|
# 3.1. use announcement data to find hidden peerings
|
||||||
for announcement in Announcement.objects.filter(router__AS__crawl=crawl):
|
for announcement in Announcement.objects.filter(router__AS__crawl=crawl):
|
||||||
path = announcement.ASPath.split(" ")
|
path = announcement.ASPath.split(" ")
|
||||||
if len(path) > 1:
|
if len(path) > 1:
|
||||||
firstASno = path.pop(0)
|
firstASno = path.pop(0)
|
||||||
firstAS = getOrCreateAS(crawl, firstASno)
|
firstAS = getOrCreateAS(crawl, firstASno)
|
||||||
while len(path) > 0:
|
while len(path) > 0:
|
||||||
secondASno = path.pop(0)
|
secondASno = path.pop(0)
|
||||||
secondAS = getOrCreateAS(crawl, secondASno)
|
secondAS = getOrCreateAS(crawl, secondASno)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
Peering.getPeering(firstAS, secondAS)
|
Peering.getPeering(firstAS, secondAS)
|
||||||
except Peering.DoesNotExist:
|
except Peering.DoesNotExist:
|
||||||
peering = Peering(as1=firstAS, as2=secondAS, origin=Peering.PATH)
|
peering = Peering(as1=firstAS, as2=secondAS, origin=Peering.PATH)
|
||||||
peering.save()
|
peering.save()
|
||||||
|
|
||||||
firstAS = secondAS
|
firstAS = secondAS
|
||||||
|
|
||||||
# 3.2 add ASses, routers and peerings from old crawlruns (last should suffice)
|
# 3.2 add ASses, routers and peerings from old crawlruns (last should suffice)
|
||||||
# find
|
# find
|
||||||
print(" --> copy old ASses")
|
print(" --> copy old ASses")
|
||||||
timerangeStart = crawl.startTime - datetime.timedelta(LAST_SEEN_DAYS)
|
timerangeStart = crawl.startTime - datetime.timedelta(LAST_SEEN_DAYS)
|
||||||
oldASses = AS.objects.filter(online=True, crawl__startTime__gte=timerangeStart).values("number").annotate(lastSeen=Max('crawl_id')).filter(~Q(lastSeen=crawl.pk))
|
oldASses = AS.objects.filter(online=True, crawl__startTime__gte=timerangeStart).values("number").annotate(lastSeen=Max('crawl_id')).filter(~Q(lastSeen=crawl.pk))
|
||||||
|
|
||||||
# 3.2.1. copy old asses
|
# 3.2.1. copy old asses
|
||||||
print(" ----> create ASses")
|
print(" ----> create ASses")
|
||||||
for oldASdata in oldASses:
|
for oldASdata in oldASses:
|
||||||
print(" ------> AS", oldASdata["number"])
|
print(" ------> AS", oldASdata["number"])
|
||||||
oldAS = AS.objects.get(number=oldASdata["number"], crawl=oldASdata["lastSeen"])
|
oldAS = AS.objects.get(number=oldASdata["number"], crawl=oldASdata["lastSeen"])
|
||||||
|
|
||||||
try:
|
try:
|
||||||
newAS = AS.objects.get(number=oldAS.number, crawl=crawl)
|
newAS = AS.objects.get(number=oldAS.number, crawl=crawl)
|
||||||
if not newAS.online and not newAS.lastSeen:
|
if not newAS.online and not newAS.lastSeen:
|
||||||
newAS.lastSeen = oldAS.crawl
|
newAS.lastSeen = oldAS.crawl
|
||||||
newAS.save()
|
newAS.save()
|
||||||
except:
|
except:
|
||||||
newAS = AS(number=oldAS.number, crawl=crawl, lastSeen=oldAS.crawl, directlyCrawled=False, online=False)
|
newAS = AS(number=oldAS.number, crawl=crawl, lastSeen=oldAS.crawl, directlyCrawled=False, online=False)
|
||||||
newAS.save()
|
newAS.save()
|
||||||
|
|
||||||
# 3.2.2 copy peerings between old asses
|
# 3.2.2 copy peerings between old asses
|
||||||
print(" ----> copy peerings")
|
print(" ----> copy peerings")
|
||||||
for oldASdata in oldASses:
|
for oldASdata in oldASses:
|
||||||
print(" ------> AS", oldASdata["number"])
|
print(" ------> AS", oldASdata["number"])
|
||||||
oldAS = AS.objects.get(number=oldASdata["number"], crawl=oldASdata["lastSeen"])
|
oldAS = AS.objects.get(number=oldASdata["number"], crawl=oldASdata["lastSeen"])
|
||||||
for peering in oldAS.getPeerings():
|
for peering in oldAS.getPeerings():
|
||||||
print(" --------> Peering %s <--> %s" % (peering.as1.number, peering.as2.number))
|
print(" --------> Peering %s <--> %s" % (peering.as1.number, peering.as2.number))
|
||||||
peering = Peering(
|
peering = Peering(
|
||||||
as1=AS.objects.get(number=peering.as1.number, crawl=crawl),
|
as1=AS.objects.get(number=peering.as1.number, crawl=crawl),
|
||||||
as2=AS.objects.get(number=peering.as2.number, crawl=crawl),
|
as2=AS.objects.get(number=peering.as2.number, crawl=crawl),
|
||||||
origin=peering.origin)
|
origin=peering.origin)
|
||||||
peering.save()
|
peering.save()
|
||||||
|
|
||||||
# 3.3 FIXME: do we also want to have old peerings which do not exist anymore?
|
# 3.3 FIXME: do we also want to have old peerings which do not exist anymore?
|
||||||
|
|
||||||
# 4. end crawl run
|
# 4. end crawl run
|
||||||
crawl.endTime = timezone.now()
|
crawl.endTime = timezone.now()
|
||||||
crawl.save()
|
crawl.save()
|
||||||
|
|
||||||
# additional data
|
# additional data
|
||||||
crawl.asCount = crawl.as_set.count()
|
crawl.asCount = crawl.as_set.count()
|
||||||
crawl.asOnlineCount = crawl.as_set.filter(online=True).count()
|
crawl.asOnlineCount = crawl.as_set.filter(online=True).count()
|
||||||
crawl.asOfflineCount = crawl.as_set.filter(online=False).count()
|
crawl.asOfflineCount = crawl.as_set.filter(online=False).count()
|
||||||
crawl.peeringCount = Peering.objects.filter(Q(as1__crawl=crawl)|Q(as2__crawl=crawl)).count()
|
crawl.peeringCount = Peering.objects.filter(Q(as1__crawl=crawl)|Q(as2__crawl=crawl)).count()
|
||||||
crawl.save()
|
crawl.save()
|
||||||
|
|
||||||
print(" !! Done")
|
print(" !! Done")
|
||||||
CrawlLog.log(crawl, "Crawl completed", severity=CrawlLog.INFO)
|
CrawlLog.log(crawl, "Crawl completed", severity=CrawlLog.INFO)
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
main()
|
main()
|
||||||
|
|
|
||||||
|
|
@ -10,344 +10,344 @@ import socket
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
|
|
||||||
class RouterParserException(Exception):
|
class RouterParserException(Exception):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def err(msg):
|
def err(msg):
|
||||||
raise RouterParserException(msg)
|
raise RouterParserException(msg)
|
||||||
|
|
||||||
|
|
||||||
def getBGPData(ip, asno):
|
def getBGPData(ip, asno):
|
||||||
rawData = getDataFromHost(ip)
|
rawData = getDataFromHost(ip)
|
||||||
if not rawData:
|
if not rawData:
|
||||||
err("Could not get data from host (empty response)")
|
err("Could not get data from host (empty response)")
|
||||||
|
|
||||||
router = parseBGPData(rawData, asno)
|
router = parseBGPData(rawData, asno)
|
||||||
|
|
||||||
router["ip"] = ip
|
router["ip"] = ip
|
||||||
|
|
||||||
return router
|
return router
|
||||||
|
|
||||||
def getDataFromHost(ip):
|
def getDataFromHost(ip):
|
||||||
socket.setdefaulttimeout(5)
|
socket.setdefaulttimeout(5)
|
||||||
x = socket.socket()
|
x = socket.socket()
|
||||||
x.connect((ip, 6556))
|
x.connect((ip, 6556))
|
||||||
f = x.makefile()
|
f = x.makefile()
|
||||||
data = f.read()
|
data = f.read()
|
||||||
x.close()
|
x.close()
|
||||||
|
|
||||||
return data
|
return data
|
||||||
|
|
||||||
def parseBGPData(raw, asno):
|
def parseBGPData(raw, asno):
|
||||||
d = re.search(r"(?:^|\n)<<<(quagga|bird)>>>\n(.*?)(?:$|<<<[^\n]+>>>)", raw, re.DOTALL)
|
d = re.search(r"(?:^|\n)<<<(quagga|bird)>>>\n(.*?)(?:$|<<<[^\n]+>>>)", raw, re.DOTALL)
|
||||||
|
|
||||||
if not d:
|
if not d:
|
||||||
err("Data not found in check mk output")
|
err("Data not found in check mk output")
|
||||||
|
|
||||||
# mkify
|
# mkify
|
||||||
raw = d.group(2).split("\n")
|
raw = d.group(2).split("\n")
|
||||||
arr = filter(lambda _z: _z, map(lambda _y: filter(lambda _x: _x, re.split(r"\s+", _y)), raw))
|
arr = filter(lambda _z: _z, map(lambda _y: filter(lambda _x: _x, re.split(r"\s+", _y)), raw))
|
||||||
|
|
||||||
# parse for bird/quagga
|
# parse for bird/quagga
|
||||||
result = None
|
result = None
|
||||||
|
|
||||||
if d.group(1) == "quagga":
|
if d.group(1) == "quagga":
|
||||||
result = parseQuagga(arr, raw, asno)
|
result = parseQuagga(arr, raw, asno)
|
||||||
else:
|
else:
|
||||||
result = parseBird(arr, raw, asno)
|
result = parseBird(arr, raw, asno)
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def parseQuagga(data, raw, asno):
|
def parseQuagga(data, raw, asno):
|
||||||
status = _quaggaFindCommand(data, "show ip bgp sum")
|
status = _quaggaFindCommand(data, "show ip bgp sum")
|
||||||
|
|
||||||
if status[0][0:3] == ['IPv4', 'Unicast', 'Summary:']:
|
if status[0][0:3] == ['IPv4', 'Unicast', 'Summary:']:
|
||||||
del(status[0])
|
del(status[0])
|
||||||
|
|
||||||
if status[0][0:3] != ['BGP', 'router', 'identifier']:
|
if status[0][0:3] != ['BGP', 'router', 'identifier']:
|
||||||
print(status)
|
print(status)
|
||||||
err("Couldn't find router id in quagga output")
|
err("Couldn't find router id in quagga output")
|
||||||
|
|
||||||
peers = _quaggaFindNeighbors(data)
|
peers = _quaggaFindNeighbors(data)
|
||||||
if asno and int(asno) != int(status[0][7]):
|
if asno and int(asno) != int(status[0][7]):
|
||||||
err("AS number (%s) does not match as number from quagga (%s)" % (asno, status[0][7]))
|
err("AS number (%s) does not match as number from quagga (%s)" % (asno, status[0][7]))
|
||||||
|
|
||||||
routes = _quaggaFindRoutes(raw)
|
routes = _quaggaFindRoutes(raw)
|
||||||
|
|
||||||
return {"local_id": status[0][3].strip(","), "local_as": int(status[0][7]), "peers": peers, "routes": routes}
|
return {"local_id": status[0][3].strip(","), "local_as": int(status[0][7]), "peers": peers, "routes": routes}
|
||||||
|
|
||||||
def parseBird(data, raw, asno):
|
def parseBird(data, raw, asno):
|
||||||
status = _birdFindTable(data, "show status")
|
status = _birdFindTable(data, "show status")
|
||||||
if status[2][0] != "1011-Router":
|
if status[2][0] != "1011-Router":
|
||||||
err("Couldn't find router id in bird output")
|
err("Couldn't find router id in bird output")
|
||||||
peers = filter(lambda _x: _x["type"] == "BGP", _birdMakeProtocols(data))
|
peers = filter(lambda _x: _x["type"] == "BGP", _birdMakeProtocols(data))
|
||||||
|
|
||||||
if asno == None:
|
if asno == None:
|
||||||
err("Host is bird")
|
err("Host is bird")
|
||||||
# FIXME
|
# FIXME
|
||||||
|
|
||||||
routes = _birdFindRoutes(data)
|
routes = _birdFindRoutes(data)
|
||||||
|
|
||||||
return {"local_id": status[2][3], "local_as": int(asno), "peers": peers, "routes": routes}
|
return {"local_id": status[2][3], "local_as": int(asno), "peers": peers, "routes": routes}
|
||||||
|
|
||||||
def _birdFindTable(info, command):
|
def _birdFindTable(info, command):
|
||||||
""" find command output of a bird command, e.g. "show bgp neighbors" """
|
""" find command output of a bird command, e.g. "show bgp neighbors" """
|
||||||
command = ["bird>"] + command.split(" ")
|
command = ["bird>"] + command.split(" ")
|
||||||
commandInfo = []
|
commandInfo = []
|
||||||
editNextLine = False
|
editNextLine = False
|
||||||
for line in info:
|
for line in info:
|
||||||
if not commandInfo:
|
if not commandInfo:
|
||||||
if line == command:
|
if line == command:
|
||||||
commandInfo.append(line)
|
commandInfo.append(line)
|
||||||
editNextLine = True
|
editNextLine = True
|
||||||
else:
|
else:
|
||||||
if editNextLine:
|
if editNextLine:
|
||||||
editNextLine = False
|
editNextLine = False
|
||||||
commandInfo.append(line[1:])
|
commandInfo.append(line[1:])
|
||||||
elif line[0] == "bird>":
|
elif line[0] == "bird>":
|
||||||
return commandInfo
|
return commandInfo
|
||||||
else:
|
else:
|
||||||
commandInfo.append(line)
|
commandInfo.append(line)
|
||||||
return []
|
return []
|
||||||
|
|
||||||
def _birdFindProtocols(info):
|
def _birdFindProtocols(info):
|
||||||
""" return a list of tuples (protoname, protoinfo) """
|
""" return a list of tuples (protoname, protoinfo) """
|
||||||
protocolTable = _birdFindTable(info, "show protocols all")
|
protocolTable = _birdFindTable(info, "show protocols all")
|
||||||
protocols = OrderedDict()
|
protocols = OrderedDict()
|
||||||
currProto = None
|
currProto = None
|
||||||
for line in protocolTable[2:]:
|
for line in protocolTable[2:]:
|
||||||
if line[0][0:4] == "1002":
|
if line[0][0:4] == "1002":
|
||||||
currProto = line[0][5:]
|
currProto = line[0][5:]
|
||||||
protocols[currProto] = [[currProto] + line[1:]]
|
protocols[currProto] = [[currProto] + line[1:]]
|
||||||
elif currProto == None:
|
elif currProto == None:
|
||||||
err("No proto selected, couldn't parse line:", line)
|
err("No proto selected, couldn't parse line:", line)
|
||||||
else:
|
else:
|
||||||
protocols[currProto].append(line)
|
protocols[currProto].append(line)
|
||||||
|
|
||||||
return protocols
|
return protocols
|
||||||
|
|
||||||
def _birdMakeProtocols(info):
|
def _birdMakeProtocols(info):
|
||||||
""" Parse birds show protocols all output """
|
""" Parse birds show protocols all output """
|
||||||
# proto: name, type, description, state (up/down?), up-since
|
# proto: name, type, description, state (up/down?), up-since
|
||||||
# routes imported, exported, preferred
|
# routes imported, exported, preferred
|
||||||
# also: routing stats (
|
# also: routing stats (
|
||||||
# bgp special stuff: state, neighbor (address, as, id) (id not available when down)
|
# bgp special stuff: state, neighbor (address, as, id) (id not available when down)
|
||||||
# state (established, active)
|
# state (established, active)
|
||||||
# if error, last error is avilable
|
# if error, last error is avilable
|
||||||
protocols = []
|
protocols = []
|
||||||
for proto, data in _birdFindProtocols(info).iteritems():
|
for proto, data in _birdFindProtocols(info).iteritems():
|
||||||
protoInfo = {
|
protoInfo = {
|
||||||
"name": proto,
|
"name": proto,
|
||||||
"type": data[0][1],
|
"type": data[0][1],
|
||||||
"table": data[0][2],
|
"table": data[0][2],
|
||||||
"state": data[0][3],
|
"state": data[0][3],
|
||||||
"last_change": data[0][4],
|
"last_change": data[0][4],
|
||||||
"info": " ".join(data[0][5:]),
|
"info": " ".join(data[0][5:]),
|
||||||
"description": " ".join(data[1][2:]),
|
"description": " ".join(data[1][2:]),
|
||||||
"routes": {
|
"routes": {
|
||||||
"imported": data[5][1],
|
"imported": data[5][1],
|
||||||
"exported": data[5][3],
|
"exported": data[5][3],
|
||||||
"preferred": data[5][5],
|
"preferred": data[5][5],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if protoInfo["type"] == "BGP":
|
if protoInfo["type"] == "BGP":
|
||||||
found = False
|
found = False
|
||||||
for n, line in enumerate(data):
|
for n, line in enumerate(data):
|
||||||
if line[0:2] == ["BGP", "state:"]:
|
if line[0:2] == ["BGP", "state:"]:
|
||||||
found = True
|
found = True
|
||||||
protoInfo["BGP"] = {
|
protoInfo["BGP"] = {
|
||||||
"state": data[n][2],
|
"state": data[n][2],
|
||||||
"online": data[n][2] == "Established",
|
"online": data[n][2] == "Established",
|
||||||
"neighbor_address": data[n+1][2],
|
"neighbor_address": data[n+1][2],
|
||||||
"neighbor_as": int(data[n+2][2]),
|
"neighbor_as": int(data[n+2][2]),
|
||||||
"neighbor_id": data[n+3][2] if len(data) > n+3 and data[n+3][0:2] == ["Neighbor", "ID:"] else None,
|
"neighbor_id": data[n+3][2] if len(data) > n+3 and data[n+3][0:2] == ["Neighbor", "ID:"] else None,
|
||||||
"last_error": " ".join(data[n+3][2:]) if len(data) > n+3 and data[n+3][0:2] == ["Last", "error:"] else None,
|
"last_error": " ".join(data[n+3][2:]) if len(data) > n+3 and data[n+3][0:2] == ["Last", "error:"] else None,
|
||||||
}
|
}
|
||||||
|
|
||||||
if not found:
|
if not found:
|
||||||
protoInfo["BGP"] = None
|
protoInfo["BGP"] = None
|
||||||
|
|
||||||
protocols.append(protoInfo)
|
protocols.append(protoInfo)
|
||||||
|
|
||||||
return protocols
|
return protocols
|
||||||
|
|
||||||
|
|
||||||
def _birdFindRoutes(info):
|
def _birdFindRoutes(info):
|
||||||
output = _birdFindTable(info, "show route all")
|
output = _birdFindTable(info, "show route all")
|
||||||
if len(output) < 1:
|
if len(output) < 1:
|
||||||
# no data found
|
# no data found
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def handleCandidate(routes, candidate):
|
def handleCandidate(routes, candidate):
|
||||||
if candidate:
|
if candidate:
|
||||||
# path, nexthop, network
|
# path, nexthop, network
|
||||||
for key in ["path", "nexthop", "network", "iBGP"]:
|
for key in ["path", "nexthop", "network", "iBGP"]:
|
||||||
if key not in candidate:
|
if key not in candidate:
|
||||||
return
|
return
|
||||||
route = {"prefix": candidate["network"], "nexthop": candidate["nexthop"], "path": candidate["path"], "iBGP": candidate["iBGP"]}
|
route = {"prefix": candidate["network"], "nexthop": candidate["nexthop"], "path": candidate["path"], "iBGP": candidate["iBGP"]}
|
||||||
routes.append(route)
|
routes.append(route)
|
||||||
pass
|
pass
|
||||||
|
|
||||||
routes = []
|
routes = []
|
||||||
candidate = None
|
candidate = None
|
||||||
lastIP = None
|
lastIP = None
|
||||||
for line in output:
|
for line in output:
|
||||||
if line[0].startswith("1007-"):
|
if line[0].startswith("1007-"):
|
||||||
# new route!
|
# new route!
|
||||||
handleCandidate(routes, candidate)
|
handleCandidate(routes, candidate)
|
||||||
if line[0] != "1007-":
|
if line[0] != "1007-":
|
||||||
# line has a network, use it!
|
# line has a network, use it!
|
||||||
lastIP = line[0][5:]
|
lastIP = line[0][5:]
|
||||||
candidate = {"network": lastIP, "iBGP": None}
|
candidate = {"network": lastIP, "iBGP": None}
|
||||||
|
|
||||||
elif candidate is not None:
|
elif candidate is not None:
|
||||||
# search bgp attributes
|
# search bgp attributes
|
||||||
if line[0] == "1012-":
|
if line[0] == "1012-":
|
||||||
pass
|
pass
|
||||||
k, v = line[1], line[2:]
|
k, v = line[1], line[2:]
|
||||||
else:
|
else:
|
||||||
k, v = line[0], line[1:]
|
k, v = line[0], line[1:]
|
||||||
|
|
||||||
k = k.rstrip(":")
|
k = k.rstrip(":")
|
||||||
if k == "BGP.next_hop":
|
if k == "BGP.next_hop":
|
||||||
candidate["nexthop"] = v[0]
|
candidate["nexthop"] = v[0]
|
||||||
elif k == "BGP.as_path":
|
elif k == "BGP.as_path":
|
||||||
candidate["path"] = v
|
candidate["path"] = v
|
||||||
|
|
||||||
|
|
||||||
handleCandidate(routes, candidate)
|
handleCandidate(routes, candidate)
|
||||||
|
|
||||||
return routes
|
return routes
|
||||||
|
|
||||||
|
|
||||||
def _quaggaFindCommand(info, cmd):
|
def _quaggaFindCommand(info, cmd):
|
||||||
# ['core-frunde#', 'show', 'ip', 'bgp', 'sum']
|
# ['core-frunde#', 'show', 'ip', 'bgp', 'sum']
|
||||||
# ['core-frunde#', 'show', 'ip', 'bgp', 'neighbors']
|
# ['core-frunde#', 'show', 'ip', 'bgp', 'neighbors']
|
||||||
output = []
|
output = []
|
||||||
cmd = cmd.split(" ")
|
cmd = cmd.split(" ")
|
||||||
prompt = None
|
prompt = None
|
||||||
for line in info:
|
for line in info:
|
||||||
if line[1:] == cmd:
|
if line[1:] == cmd:
|
||||||
prompt = line[0]
|
prompt = line[0]
|
||||||
elif line[0] == prompt:
|
elif line[0] == prompt:
|
||||||
# done
|
# done
|
||||||
return output
|
return output
|
||||||
elif prompt != None:
|
elif prompt != None:
|
||||||
output.append(line)
|
output.append(line)
|
||||||
|
|
||||||
err("Could not find command '%s' in output" % " ".join(cmd))
|
err("Could not find command '%s' in output" % " ".join(cmd))
|
||||||
|
|
||||||
def _quaggaFindNeighbors(info):
|
def _quaggaFindNeighbors(info):
|
||||||
#['BGP', 'neighbor', 'is', '10.50.1.2,', 'remote', 'AS', '65001,', 'local', 'AS', '65001,', 'internal', 'link']
|
#['BGP', 'neighbor', 'is', '10.50.1.2,', 'remote', 'AS', '65001,', 'local', 'AS', '65001,', 'internal', 'link']
|
||||||
output = _quaggaFindCommand(info, "show ip bgp neighbors")
|
output = _quaggaFindCommand(info, "show ip bgp neighbors")
|
||||||
start = ["BGP", "neighbor", "is"]
|
start = ["BGP", "neighbor", "is"]
|
||||||
|
|
||||||
curr = None
|
curr = None
|
||||||
rawNeighbors = []
|
rawNeighbors = []
|
||||||
for line in output:
|
for line in output:
|
||||||
if line[0:3] == start:
|
if line[0:3] == start:
|
||||||
if curr:
|
if curr:
|
||||||
rawNeighbors.append(curr)
|
rawNeighbors.append(curr)
|
||||||
curr = [line]
|
curr = [line]
|
||||||
elif curr:
|
elif curr:
|
||||||
curr.append(line)
|
curr.append(line)
|
||||||
else:
|
else:
|
||||||
err("Could not find start of neighbors")
|
err("Could not find start of neighbors")
|
||||||
|
|
||||||
if curr:
|
if curr:
|
||||||
rawNeighbors.append(curr)
|
rawNeighbors.append(curr)
|
||||||
curr = None
|
curr = None
|
||||||
|
|
||||||
neighbors = []
|
neighbors = []
|
||||||
neighborDict = OrderedDict()
|
neighborDict = OrderedDict()
|
||||||
for raw in rawNeighbors:
|
for raw in rawNeighbors:
|
||||||
descrIdx = 1 if raw[1][0] == "Description:" else 0
|
descrIdx = 1 if raw[1][0] == "Description:" else 0
|
||||||
if raw[descrIdx + 1][0] == "Hostname:":
|
if raw[descrIdx + 1][0] == "Hostname:":
|
||||||
descrIdx += 1
|
descrIdx += 1
|
||||||
|
|
||||||
peerdict = {
|
peerdict = {
|
||||||
"neighbor_address": raw[0][3].rstrip(","),
|
"neighbor_address": raw[0][3].rstrip(","),
|
||||||
"neighbor_as": int(raw[0][6].rstrip(",")),
|
"neighbor_as": int(raw[0][6].rstrip(",")),
|
||||||
"local_as": int(raw[0][9].rstrip(",")),
|
"local_as": int(raw[0][9].rstrip(",")),
|
||||||
"description": " ".join(raw[1][1:]) if descrIdx else "No description",
|
"description": " ".join(raw[1][1:]) if descrIdx else "No description",
|
||||||
"neighbor_id": raw[1+descrIdx][6].strip(","),
|
"neighbor_id": raw[1+descrIdx][6].strip(","),
|
||||||
"state": raw[2+descrIdx][3].strip(","),
|
"state": raw[2+descrIdx][3].strip(","),
|
||||||
"routes": {
|
"routes": {
|
||||||
"imported": 0,
|
"imported": 0,
|
||||||
},
|
},
|
||||||
"BGP": {
|
"BGP": {
|
||||||
"state": raw[2+descrIdx][3].strip(","),
|
"state": raw[2+descrIdx][3].strip(","),
|
||||||
"online": raw[2+descrIdx][3].strip(",") == "Established",
|
"online": raw[2+descrIdx][3].strip(",") == "Established",
|
||||||
"neighbor_id": raw[1+descrIdx][6].strip(","),
|
"neighbor_id": raw[1+descrIdx][6].strip(","),
|
||||||
"neighbor_address": raw[0][3].rstrip(","),
|
"neighbor_address": raw[0][3].rstrip(","),
|
||||||
"neighbor_as": int(raw[0][6].rstrip(",")),
|
"neighbor_as": int(raw[0][6].rstrip(",")),
|
||||||
"state": raw[2+descrIdx][3].strip(","),
|
"state": raw[2+descrIdx][3].strip(","),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for line in raw:
|
for line in raw:
|
||||||
if line[1:3] == ["accepted", "prefixes"]:
|
if line[1:3] == ["accepted", "prefixes"]:
|
||||||
# woooo
|
# woooo
|
||||||
peerdict["routes"]["imported"] = int(line[0])
|
peerdict["routes"]["imported"] = int(line[0])
|
||||||
break
|
break
|
||||||
|
|
||||||
neighbors.append(peerdict)
|
neighbors.append(peerdict)
|
||||||
neighborDict[peerdict["neighbor_address"]] = peerdict
|
neighborDict[peerdict["neighbor_address"]] = peerdict
|
||||||
|
|
||||||
return neighbors
|
return neighbors
|
||||||
|
|
||||||
def _quaggaFindRoutes(raw):
|
def _quaggaFindRoutes(raw):
|
||||||
# from # show ip bgp to Total number of prefixes XX
|
# from # show ip bgp to Total number of prefixes XX
|
||||||
# BGP table version is 0, local router ID is 10.50.0.1
|
# BGP table version is 0, local router ID is 10.50.0.1
|
||||||
# *> 10.3.14.0/27 10.75.0.22 0 65002 65112 i
|
# *> 10.3.14.0/27 10.75.0.22 0 65002 65112 i
|
||||||
cmdre = re.compile(r"^([^\s#]+#) show ip bgp$")
|
cmdre = re.compile(r"^([^\s#]+#) show ip bgp$")
|
||||||
routere = re.compile(r"^(?P<status>.)(?P<status2>.)(?P<origin>.)(?P<network>[0-9./]+)?\s+(?P<nexthop>[0-9./]+)[\s0-9i?]+$")
|
routere = re.compile(r"^(?P<status>.)(?P<status2>.)(?P<origin>.)(?P<network>[0-9./]+)?\s+(?P<nexthop>[0-9./]+)[\s0-9i?]+$")
|
||||||
|
|
||||||
# find output
|
# find output
|
||||||
output = []
|
output = []
|
||||||
prompt = None
|
prompt = None
|
||||||
for line in raw:
|
for line in raw:
|
||||||
if not prompt:
|
if not prompt:
|
||||||
m = cmdre.match(line)
|
m = cmdre.match(line)
|
||||||
if m:
|
if m:
|
||||||
prompt = m.group(1) + " "
|
prompt = m.group(1) + " "
|
||||||
else:
|
else:
|
||||||
if line.startswith(prompt):
|
if line.startswith(prompt):
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
output.append(line)
|
output.append(line)
|
||||||
|
|
||||||
if len(output) < 1:
|
if len(output) < 1:
|
||||||
# no data found
|
# no data found
|
||||||
return None
|
return None
|
||||||
|
|
||||||
routes = []
|
routes = []
|
||||||
foundTable = False
|
foundTable = False
|
||||||
lastIP = None
|
lastIP = None
|
||||||
for line in output:
|
for line in output:
|
||||||
if not foundTable:
|
if not foundTable:
|
||||||
if line.endswith("Metric LocPrf Weight Path"):
|
if line.endswith("Metric LocPrf Weight Path"):
|
||||||
foundTable = True
|
foundTable = True
|
||||||
else:
|
else:
|
||||||
if line != '':
|
if line != '':
|
||||||
if line.startswith("Total number of prefixes") or line.startswith("Displayed "):
|
if line.startswith("Total number of prefixes") or line.startswith("Displayed "):
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
# parse one route line
|
# parse one route line
|
||||||
#print(line)
|
#print(line)
|
||||||
m = routere.match(line)
|
m = routere.match(line)
|
||||||
d = m.groupdict()
|
d = m.groupdict()
|
||||||
if d["network"]:
|
if d["network"]:
|
||||||
lastIP = d["network"]
|
lastIP = d["network"]
|
||||||
else:
|
else:
|
||||||
d["network"] = lastIP
|
d["network"] = lastIP
|
||||||
|
|
||||||
# "parse" path (everything after 61 chars, but no i)
|
# "parse" path (everything after 61 chars, but no i)
|
||||||
path = filter(lambda _x: _x not in ('', 'i'), line[61:].split(" "))
|
path = filter(lambda _x: _x not in ('', 'i'), line[61:].split(" "))
|
||||||
|
|
||||||
# currently skip incomplete routes
|
# currently skip incomplete routes
|
||||||
if '?' not in path:
|
if '?' not in path:
|
||||||
route = {"prefix": d["network"], "nexthop": d["nexthop"], "path": path, "iBGP": d["origin"] == "i"}
|
route = {"prefix": d["network"], "nexthop": d["nexthop"], "path": path, "iBGP": d["origin"] == "i"}
|
||||||
routes.append(route)
|
routes.append(route)
|
||||||
|
|
||||||
return routes
|
return routes
|
||||||
|
|
|
||||||
|
|
@ -30,7 +30,7 @@ TEMPLATE_DEBUG = True
|
||||||
ALLOWED_HOSTS = []
|
ALLOWED_HOSTS = []
|
||||||
|
|
||||||
STATICFILES_DIRS = (
|
STATICFILES_DIRS = (
|
||||||
'static/',
|
'static/',
|
||||||
)
|
)
|
||||||
|
|
||||||
# Application definition
|
# Application definition
|
||||||
|
|
@ -42,8 +42,8 @@ INSTALLED_APPS = (
|
||||||
'django.contrib.sessions',
|
'django.contrib.sessions',
|
||||||
'django.contrib.messages',
|
'django.contrib.messages',
|
||||||
'django.contrib.staticfiles',
|
'django.contrib.staticfiles',
|
||||||
'bgpdata',
|
'bgpdata',
|
||||||
'tastypie',
|
'tastypie',
|
||||||
)
|
)
|
||||||
|
|
||||||
API_LIMIT_PER_PAGE = 100
|
API_LIMIT_PER_PAGE = 100
|
||||||
|
|
@ -86,7 +86,7 @@ USE_L10N = True
|
||||||
|
|
||||||
USE_TZ = True
|
USE_TZ = True
|
||||||
TEMPLATE_DIRS = (
|
TEMPLATE_DIRS = (
|
||||||
'templates/',
|
'templates/',
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -29,7 +29,7 @@ DEBUG = True
|
||||||
ALLOWED_HOSTS = []
|
ALLOWED_HOSTS = []
|
||||||
|
|
||||||
STATICFILES_DIRS = (
|
STATICFILES_DIRS = (
|
||||||
os.path.join(BASE_DIR, 'static/'),
|
os.path.join(BASE_DIR, 'static/'),
|
||||||
)
|
)
|
||||||
|
|
||||||
# Application definition
|
# Application definition
|
||||||
|
|
@ -41,8 +41,8 @@ INSTALLED_APPS = (
|
||||||
'django.contrib.sessions',
|
'django.contrib.sessions',
|
||||||
'django.contrib.messages',
|
'django.contrib.messages',
|
||||||
'django.contrib.staticfiles',
|
'django.contrib.staticfiles',
|
||||||
'bgpdata',
|
'bgpdata',
|
||||||
'tastypie',
|
'tastypie',
|
||||||
)
|
)
|
||||||
|
|
||||||
API_LIMIT_PER_PAGE = 100
|
API_LIMIT_PER_PAGE = 100
|
||||||
|
|
|
||||||
|
|
@ -11,8 +11,8 @@ urlpatterns = (
|
||||||
# Examples:
|
# Examples:
|
||||||
# url(r'^$', 'dnmapper.views.home', name='home'),
|
# url(r'^$', 'dnmapper.views.home', name='home'),
|
||||||
# url(r'^blog/', include('blog.urls')),
|
# url(r'^blog/', include('blog.urls')),
|
||||||
url(r'^$', RedirectView.as_view(url='/map/')),
|
url(r'^$', RedirectView.as_view(url='/map/')),
|
||||||
url(r'^map/', include(bgpdata.urls)),
|
url(r'^map/', include(bgpdata.urls)),
|
||||||
|
|
||||||
url(r'^admin/', include(admin.site.urls)),
|
url(r'^admin/', include(admin.site.urls)),
|
||||||
)
|
)
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue