diff --git a/bgpdata/models.py b/bgpdata/models.py index 6551de7..a9e75a1 100644 --- a/bgpdata/models.py +++ b/bgpdata/models.py @@ -69,9 +69,18 @@ class AS(models.Model): online = models.BooleanField(default=True) lastSeen = models.ForeignKey(CrawlRun, blank=True, null=True, default=None, related_name='as_lastseen') + class Meta: + unique_together = (('crawl', 'number'),) + def __unicode__(self): return u"AS %s (crawl %d)" % (self.number, self.crawl.pk) + def setOnline(self): + if not self.online: + self.online = True + self.lastSeen = None + self.save() + def getPeerings(self): return Peering.objects.filter(Q(as1=self)|Q(as2=self)) diff --git a/bin/crawl.py b/bin/crawl.py index 72188d3..8f5046e 100755 --- a/bin/crawl.py +++ b/bin/crawl.py @@ -1,6 +1,9 @@ #!/usr/bin/env python2 from __future__ import print_function +# config +LAST_SEEN_DAYS = 7 + # prepare environment import sys sys.path.append("..") @@ -21,6 +24,8 @@ def getOrCreateAS(crawl, number, online=True): currAS = None try: currAS = AS.objects.get(crawl=crawl, number=number) + if online: + currAS.setOnline() except AS.DoesNotExist: currAS = AS(crawl=crawl, number=number, online=online) currAS.save() @@ -76,12 +81,7 @@ def main(): # a) find/create neighbor print(" ----> Peer:", int(peer["BGP"]["neighbor_as"])) - neighAS = None - try: - neighAS = AS.objects.get(crawl=crawl, number=int(peer["BGP"]["neighbor_as"])) - except AS.DoesNotExist: - neighAS = AS(crawl=crawl, number=int(peer["BGP"]["neighbor_as"])) - neighAS.save() + neighAS = getOrCreateAS(crawl, int(peer["BGP"]["neighbor_as"]), online=peer["BGP"]["online"]) # b) find out if a peering already exists (maybe where we only need to add our router id?) peering = None @@ -155,8 +155,8 @@ def main(): # 3.2 add ASses, routers and peerings from old crawlruns (last should suffice) # find print(" --> copy old ASses") - timerangeStart = crawl.startTime - datetime.timedelta(7) - oldASses = AS.objects.filter(crawl__startTime__gte=timerangeStart).values("number").annotate(lastSeen=Max('crawl_id')).filter(~Q(lastSeen=crawl.pk)) + timerangeStart = crawl.startTime - datetime.timedelta(LAST_SEEN_DAYS) + oldASses = AS.objects.filter(online=True, crawl__startTime__gte=timerangeStart).values("number").annotate(lastSeen=Max('crawl_id')).filter(~Q(lastSeen=crawl.pk)) # 3.2.1. copy old asses print(" ----> create ASses") @@ -164,8 +164,14 @@ def main(): print(" ------> AS", oldASdata["number"]) oldAS = AS.objects.get(number=oldASdata["number"], crawl=oldASdata["lastSeen"]) - newAS = AS(number=oldAS.number, crawl=crawl, lastSeen=oldAS.crawl, directlyCrawled=False, online=False) - newAS.save() + try: + newAS = AS.objects.get(number=oldAS.number, crawl=crawl) + if not newAS.online and not newAS.lastSeen: + newAS.lastSeen = oldAS.crawl + newAS.save() + except: + newAS = AS(number=oldAS.number, crawl=crawl, lastSeen=oldAS.crawl, directlyCrawled=False, online=False) + newAS.save() # 3.2.2 copy peerings between old asses print(" ----> copy peerings") diff --git a/bin/routerparsers.py b/bin/routerparsers.py index 096d37a..f03b3f9 100644 --- a/bin/routerparsers.py +++ b/bin/routerparsers.py @@ -151,6 +151,7 @@ def _birdMakeProtocols(info): found = True protoInfo["BGP"] = { "state": data[n][2], + "online": data[n][2] == "Established", "neighbor_address": data[n+1][2], "neighbor_as": int(data[n+2][2]), "neighbor_id": data[n+3][2] if data[n+3][0:2] == ["Neighbor", "ID:"] else None, @@ -267,6 +268,7 @@ def _quaggaFindNeighbors(info): }, "BGP": { "state": raw[2+descrIdx][3].strip(","), + "online": raw[2+descrIdx][3].strip(",") == "Established", "neighbor_id": raw[1+descrIdx][6].strip(","), "neighbor_address": raw[0][3].rstrip(","), "neighbor_as": int(raw[0][6].rstrip(",")),