Show peers as offline, if they are

bgpdata/models.py

@@ -69,9 +69,18 @@ class AS(models.Model):
 	online = models.BooleanField(default=True)
 	lastSeen = models.ForeignKey(CrawlRun, blank=True, null=True, default=None, related_name='as_lastseen')
 
+	class Meta:
+		unique_together = (('crawl', 'number'),)
+
 	def __unicode__(self):
 		return u"AS %s (crawl %d)" % (self.number, self.crawl.pk)
 
+	def setOnline(self):
+		if not self.online:
+			self.online = True
+			self.lastSeen = None
+			self.save()
+
 	def getPeerings(self):
 		return Peering.objects.filter(Q(as1=self)|Q(as2=self))
 

bin/crawl.py

@@ -1,6 +1,9 @@
 #!/usr/bin/env python2
 from __future__ import print_function
 
+# config
+LAST_SEEN_DAYS = 7
+
 # prepare environment
 import sys
 sys.path.append("..")
@@ -21,6 +24,8 @@ def getOrCreateAS(crawl, number, online=True):
 	currAS = None
 	try:
 		currAS = AS.objects.get(crawl=crawl, number=number)
+		if online:
+			currAS.setOnline()
 	except AS.DoesNotExist:
 		currAS = AS(crawl=crawl, number=number, online=online)
 		currAS.save()
@@ -76,12 +81,7 @@ def main():
 
 			# a) find/create neighbor
 			print(" ----> Peer:", int(peer["BGP"]["neighbor_as"]))
-			neighAS = None
+			neighAS = getOrCreateAS(crawl, int(peer["BGP"]["neighbor_as"]), online=peer["BGP"]["online"])
-			try:
-				neighAS = AS.objects.get(crawl=crawl, number=int(peer["BGP"]["neighbor_as"]))
-			except AS.DoesNotExist:
-				neighAS = AS(crawl=crawl, number=int(peer["BGP"]["neighbor_as"]))
-				neighAS.save()
 
 			# b) find out if a peering already exists (maybe where we only need to add our router id?)
 			peering = None
@@ -155,8 +155,8 @@ def main():
 	# 3.2 add ASses, routers and peerings from old crawlruns (last should suffice)
 	# find 
 	print(" --> copy old ASses")
-	timerangeStart = crawl.startTime - datetime.timedelta(7)
+	timerangeStart = crawl.startTime - datetime.timedelta(LAST_SEEN_DAYS)
-	oldASses = AS.objects.filter(crawl__startTime__gte=timerangeStart).values("number").annotate(lastSeen=Max('crawl_id')).filter(~Q(lastSeen=crawl.pk))
+	oldASses = AS.objects.filter(online=True, crawl__startTime__gte=timerangeStart).values("number").annotate(lastSeen=Max('crawl_id')).filter(~Q(lastSeen=crawl.pk))
 
 	# 3.2.1. copy old asses
 	print(" ----> create ASses")
@@ -164,8 +164,14 @@ def main():
 		print(" ------> AS", oldASdata["number"])
 		oldAS = AS.objects.get(number=oldASdata["number"], crawl=oldASdata["lastSeen"])
 
-		newAS = AS(number=oldAS.number, crawl=crawl, lastSeen=oldAS.crawl, directlyCrawled=False, online=False)
+		try:
-		newAS.save()
+			newAS = AS.objects.get(number=oldAS.number, crawl=crawl)
+			if not newAS.online and not newAS.lastSeen:
+				newAS.lastSeen = oldAS.crawl
+				newAS.save()
+		except:
+			newAS = AS(number=oldAS.number, crawl=crawl, lastSeen=oldAS.crawl, directlyCrawled=False, online=False)
+			newAS.save()
 
 	# 3.2.2 copy peerings between old asses
 	print(" ----> copy peerings")

bin/routerparsers.py

@@ -151,6 +151,7 @@ def _birdMakeProtocols(info):
 					found = True
 					protoInfo["BGP"] = {
 						"state": data[n][2],
+						"online": data[n][2] == "Established",
 						"neighbor_address": data[n+1][2],
 						"neighbor_as": int(data[n+2][2]),
 						"neighbor_id": data[n+3][2] if data[n+3][0:2] == ["Neighbor", "ID:"] else None,
@@ -267,6 +268,7 @@ def _quaggaFindNeighbors(info):
 			},
 			"BGP": {
 				"state": raw[2+descrIdx][3].strip(","),
+				"online": raw[2+descrIdx][3].strip(",") == "Established",
 				"neighbor_id": raw[1+descrIdx][6].strip(","),
 				"neighbor_address": raw[0][3].rstrip(","),
 				"neighbor_as": int(raw[0][6].rstrip(",")),