|
|
|
@ -26,193 +26,193 @@ from routerparsers import getBGPData, RouterParserException
@@ -26,193 +26,193 @@ from routerparsers import getBGPData, RouterParserException
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def getOrCreateAS(crawl, number, online=True): |
|
|
|
|
currAS = None |
|
|
|
|
try: |
|
|
|
|
currAS = AS.objects.get(crawl=crawl, number=number) |
|
|
|
|
if online: |
|
|
|
|
currAS.setOnline() |
|
|
|
|
except AS.DoesNotExist: |
|
|
|
|
currAS = AS(crawl=crawl, number=number, online=online) |
|
|
|
|
currAS.save() |
|
|
|
|
currAS = None |
|
|
|
|
try: |
|
|
|
|
currAS = AS.objects.get(crawl=crawl, number=number) |
|
|
|
|
if online: |
|
|
|
|
currAS.setOnline() |
|
|
|
|
except AS.DoesNotExist: |
|
|
|
|
currAS = AS(crawl=crawl, number=number, online=online) |
|
|
|
|
currAS.save() |
|
|
|
|
|
|
|
|
|
return currAS |
|
|
|
|
return currAS |
|
|
|
|
|
|
|
|
|
def main(): |
|
|
|
|
# 1. create crawl run |
|
|
|
|
crawl = CrawlRun() |
|
|
|
|
crawl.startTime = timezone.now() |
|
|
|
|
crawl.save() |
|
|
|
|
|
|
|
|
|
CrawlLog.log(crawl, "Starting crawl run!", severity=CrawlLog.INFO) |
|
|
|
|
|
|
|
|
|
# 2. get data from all hosts, put it in the database |
|
|
|
|
for host in ConfigHost.objects.all(): |
|
|
|
|
crawl.hostsCrawled.add(host) |
|
|
|
|
data = None |
|
|
|
|
print(" -- Getting data for host %s" % host) |
|
|
|
|
try: |
|
|
|
|
if host.checkMethod == 'CMK': |
|
|
|
|
data = getBGPData(host.ip, host.number) |
|
|
|
|
else: |
|
|
|
|
CrawlLog.log(crawl, "Method %s is not currently supported, skipping host" % host.checkMethod, host=host, severity=CrawlLog.ERROR) |
|
|
|
|
continue |
|
|
|
|
except RouterParserException as e: |
|
|
|
|
msg = "Could not parse data for host: %s" % str(e) |
|
|
|
|
print("%s: %s" % (host, msg)) |
|
|
|
|
CrawlLog.log(crawl, msg, host=host, severity=CrawlLog.ERROR) |
|
|
|
|
continue |
|
|
|
|
except socket.error as e: |
|
|
|
|
msg = "Could not reach host: %s" % (e,) |
|
|
|
|
print("%s: %s" % (host, msg)) |
|
|
|
|
CrawlLog.log(crawl, msg, host=host, severity=CrawlLog.ERROR) |
|
|
|
|
continue |
|
|
|
|
|
|
|
|
|
print(" -- parsing...") |
|
|
|
|
|
|
|
|
|
currASno = int(data["local_as"]) |
|
|
|
|
currAS = getOrCreateAS(crawl, currASno) |
|
|
|
|
|
|
|
|
|
currAS.directlyCrawled = True |
|
|
|
|
currAS.save() |
|
|
|
|
|
|
|
|
|
currRouter = None |
|
|
|
|
try: |
|
|
|
|
currRouter = BorderRouter.objects.get(AS=currAS, routerID=data["local_id"]) |
|
|
|
|
currRouter.pingable = True |
|
|
|
|
currRouter.reachable = True |
|
|
|
|
currRouter.save() |
|
|
|
|
except BorderRouter.DoesNotExist: |
|
|
|
|
currRouter = BorderRouter(AS=currAS, routerID=data["local_id"], pingable=True, reachable=True) |
|
|
|
|
currRouter.save() |
|
|
|
|
|
|
|
|
|
print(" --> peers") |
|
|
|
|
for peer in data["peers"]: |
|
|
|
|
# peerings |
|
|
|
|
# data: BGP{state, neighbor_id, neighbor_as}, description |
|
|
|
|
|
|
|
|
|
# a) find/create neighbor |
|
|
|
|
print(" ----> Peer:", int(peer["BGP"]["neighbor_as"])) |
|
|
|
|
neighAS = getOrCreateAS(crawl, int(peer["BGP"]["neighbor_as"]), online=peer["BGP"]["online"]) |
|
|
|
|
|
|
|
|
|
# b) find out if a peering already exists (maybe where we only need to add our router id?) |
|
|
|
|
peering = None |
|
|
|
|
try: |
|
|
|
|
peering = Peering.getPeering(currAS, neighAS) |
|
|
|
|
except Peering.DoesNotExist: |
|
|
|
|
peering = Peering(as1=currAS, as2=neighAS, origin=Peering.DIRECT) |
|
|
|
|
peering.save() |
|
|
|
|
|
|
|
|
|
# c) look for router/peering pairs |
|
|
|
|
if peer["BGP"]["neighbor_id"]: |
|
|
|
|
try: |
|
|
|
|
neighRouter = BorderRouter.objects.get(AS=neighAS, routerID=peer["BGP"]["neighbor_id"]) |
|
|
|
|
except BorderRouter.DoesNotExist: |
|
|
|
|
neighRouter = BorderRouter(AS=neighAS, routerID=peer["BGP"]["neighbor_id"], pingable=False, reachable=False) |
|
|
|
|
neighRouter.save() |
|
|
|
|
try: |
|
|
|
|
BorderRouterPair.getPairing(peering, currRouter, neighRouter) |
|
|
|
|
except BorderRouterPair.DoesNotExist: |
|
|
|
|
pairs = BorderRouterPair.objects.filter(Q(peering=peering) & (Q(router1=neighRouter, router2=None)|Q(router1=None, router2=neighRouter))) |
|
|
|
|
if pairs.count() > 0: |
|
|
|
|
pair = pairs[0] |
|
|
|
|
if pair.router1 == None: |
|
|
|
|
pair.router1 = currRouter |
|
|
|
|
else: |
|
|
|
|
pair.router2 = currRouter |
|
|
|
|
pair.save() |
|
|
|
|
else: |
|
|
|
|
pair = BorderRouterPair(peering=peering, router1=currRouter, router2=neighRouter) |
|
|
|
|
pair.save() |
|
|
|
|
|
|
|
|
|
print(" --> Announcements") |
|
|
|
|
if "routes" in data and data["routes"]: |
|
|
|
|
for route in data["routes"]: |
|
|
|
|
print(" ---->", route["prefix"]) |
|
|
|
|
if "/" not in route["prefix"]: |
|
|
|
|
continue |
|
|
|
|
|
|
|
|
|
crawlAS = currAS |
|
|
|
|
if len(route["path"]) > 0: |
|
|
|
|
crawlAS = getOrCreateAS(crawl, route["path"][0]) |
|
|
|
|
originAS = getOrCreateAS(crawl, route["path"][-1]) |
|
|
|
|
ip, prefix = route["prefix"].split("/") |
|
|
|
|
a = Announcement(router=currRouter, ip=ip, prefix=prefix, |
|
|
|
|
ASPath=" ".join(route["path"]), nextHop=route["nexthop"], |
|
|
|
|
crawlAS=crawlAS, originAS=originAS) |
|
|
|
|
a.save() |
|
|
|
|
else: |
|
|
|
|
print(" !! No routes found in host output") |
|
|
|
|
CrawlLog.log(crawl, "No routes found in host output (no bgp feed included?)", host=host, severity=CrawlLog.WARN) |
|
|
|
|
|
|
|
|
|
# 3. calculate missing data |
|
|
|
|
print(" -- Adding extra data from announcements...") |
|
|
|
|
# 3.1. use announcement data to find hidden peerings |
|
|
|
|
for announcement in Announcement.objects.filter(router__AS__crawl=crawl): |
|
|
|
|
path = announcement.ASPath.split(" ") |
|
|
|
|
if len(path) > 1: |
|
|
|
|
firstASno = path.pop(0) |
|
|
|
|
firstAS = getOrCreateAS(crawl, firstASno) |
|
|
|
|
while len(path) > 0: |
|
|
|
|
secondASno = path.pop(0) |
|
|
|
|
secondAS = getOrCreateAS(crawl, secondASno) |
|
|
|
|
|
|
|
|
|
try: |
|
|
|
|
Peering.getPeering(firstAS, secondAS) |
|
|
|
|
except Peering.DoesNotExist: |
|
|
|
|
peering = Peering(as1=firstAS, as2=secondAS, origin=Peering.PATH) |
|
|
|
|
peering.save() |
|
|
|
|
|
|
|
|
|
firstAS = secondAS |
|
|
|
|
|
|
|
|
|
# 3.2 add ASses, routers and peerings from old crawlruns (last should suffice) |
|
|
|
|
# find |
|
|
|
|
print(" --> copy old ASses") |
|
|
|
|
timerangeStart = crawl.startTime - datetime.timedelta(LAST_SEEN_DAYS) |
|
|
|
|
oldASses = AS.objects.filter(online=True, crawl__startTime__gte=timerangeStart).values("number").annotate(lastSeen=Max('crawl_id')).filter(~Q(lastSeen=crawl.pk)) |
|
|
|
|
|
|
|
|
|
# 3.2.1. copy old asses |
|
|
|
|
print(" ----> create ASses") |
|
|
|
|
for oldASdata in oldASses: |
|
|
|
|
print(" ------> AS", oldASdata["number"]) |
|
|
|
|
oldAS = AS.objects.get(number=oldASdata["number"], crawl=oldASdata["lastSeen"]) |
|
|
|
|
|
|
|
|
|
try: |
|
|
|
|
newAS = AS.objects.get(number=oldAS.number, crawl=crawl) |
|
|
|
|
if not newAS.online and not newAS.lastSeen: |
|
|
|
|
newAS.lastSeen = oldAS.crawl |
|
|
|
|
newAS.save() |
|
|
|
|
except: |
|
|
|
|
newAS = AS(number=oldAS.number, crawl=crawl, lastSeen=oldAS.crawl, directlyCrawled=False, online=False) |
|
|
|
|
newAS.save() |
|
|
|
|
|
|
|
|
|
# 3.2.2 copy peerings between old asses |
|
|
|
|
print(" ----> copy peerings") |
|
|
|
|
for oldASdata in oldASses: |
|
|
|
|
print(" ------> AS", oldASdata["number"]) |
|
|
|
|
oldAS = AS.objects.get(number=oldASdata["number"], crawl=oldASdata["lastSeen"]) |
|
|
|
|
for peering in oldAS.getPeerings(): |
|
|
|
|
print(" --------> Peering %s <--> %s" % (peering.as1.number, peering.as2.number)) |
|
|
|
|
peering = Peering( |
|
|
|
|
as1=AS.objects.get(number=peering.as1.number, crawl=crawl), |
|
|
|
|
as2=AS.objects.get(number=peering.as2.number, crawl=crawl), |
|
|
|
|
origin=peering.origin) |
|
|
|
|
peering.save() |
|
|
|
|
|
|
|
|
|
# 3.3 FIXME: do we also want to have old peerings which do not exist anymore? |
|
|
|
|
|
|
|
|
|
# 4. end crawl run |
|
|
|
|
crawl.endTime = timezone.now() |
|
|
|
|
crawl.save() |
|
|
|
|
|
|
|
|
|
# additional data |
|
|
|
|
crawl.asCount = crawl.as_set.count() |
|
|
|
|
crawl.asOnlineCount = crawl.as_set.filter(online=True).count() |
|
|
|
|
crawl.asOfflineCount = crawl.as_set.filter(online=False).count() |
|
|
|
|
crawl.peeringCount = Peering.objects.filter(Q(as1__crawl=crawl)|Q(as2__crawl=crawl)).count() |
|
|
|
|
crawl.save() |
|
|
|
|
|
|
|
|
|
print(" !! Done") |
|
|
|
|
CrawlLog.log(crawl, "Crawl completed", severity=CrawlLog.INFO) |
|
|
|
|
# 1. create crawl run |
|
|
|
|
crawl = CrawlRun() |
|
|
|
|
crawl.startTime = timezone.now() |
|
|
|
|
crawl.save() |
|
|
|
|
|
|
|
|
|
CrawlLog.log(crawl, "Starting crawl run!", severity=CrawlLog.INFO) |
|
|
|
|
|
|
|
|
|
# 2. get data from all hosts, put it in the database |
|
|
|
|
for host in ConfigHost.objects.all(): |
|
|
|
|
crawl.hostsCrawled.add(host) |
|
|
|
|
data = None |
|
|
|
|
print(" -- Getting data for host %s" % host) |
|
|
|
|
try: |
|
|
|
|
if host.checkMethod == 'CMK': |
|
|
|
|
data = getBGPData(host.ip, host.number) |
|
|
|
|
else: |
|
|
|
|
CrawlLog.log(crawl, "Method %s is not currently supported, skipping host" % host.checkMethod, host=host, severity=CrawlLog.ERROR) |
|
|
|
|
continue |
|
|
|
|
except RouterParserException as e: |
|
|
|
|
msg = "Could not parse data for host: %s" % str(e) |
|
|
|
|
print("%s: %s" % (host, msg)) |
|
|
|
|
CrawlLog.log(crawl, msg, host=host, severity=CrawlLog.ERROR) |
|
|
|
|
continue |
|
|
|
|
except socket.error as e: |
|
|
|
|
msg = "Could not reach host: %s" % (e,) |
|
|
|
|
print("%s: %s" % (host, msg)) |
|
|
|
|
CrawlLog.log(crawl, msg, host=host, severity=CrawlLog.ERROR) |
|
|
|
|
continue |
|
|
|
|
|
|
|
|
|
print(" -- parsing...") |
|
|
|
|
|
|
|
|
|
currASno = int(data["local_as"]) |
|
|
|
|
currAS = getOrCreateAS(crawl, currASno) |
|
|
|
|
|
|
|
|
|
currAS.directlyCrawled = True |
|
|
|
|
currAS.save() |
|
|
|
|
|
|
|
|
|
currRouter = None |
|
|
|
|
try: |
|
|
|
|
currRouter = BorderRouter.objects.get(AS=currAS, routerID=data["local_id"]) |
|
|
|
|
currRouter.pingable = True |
|
|
|
|
currRouter.reachable = True |
|
|
|
|
currRouter.save() |
|
|
|
|
except BorderRouter.DoesNotExist: |
|
|
|
|
currRouter = BorderRouter(AS=currAS, routerID=data["local_id"], pingable=True, reachable=True) |
|
|
|
|
currRouter.save() |
|
|
|
|
|
|
|
|
|
print(" --> peers") |
|
|
|
|
for peer in data["peers"]: |
|
|
|
|
# peerings |
|
|
|
|
# data: BGP{state, neighbor_id, neighbor_as}, description |
|
|
|
|
|
|
|
|
|
# a) find/create neighbor |
|
|
|
|
print(" ----> Peer:", int(peer["BGP"]["neighbor_as"])) |
|
|
|
|
neighAS = getOrCreateAS(crawl, int(peer["BGP"]["neighbor_as"]), online=peer["BGP"]["online"]) |
|
|
|
|
|
|
|
|
|
# b) find out if a peering already exists (maybe where we only need to add our router id?) |
|
|
|
|
peering = None |
|
|
|
|
try: |
|
|
|
|
peering = Peering.getPeering(currAS, neighAS) |
|
|
|
|
except Peering.DoesNotExist: |
|
|
|
|
peering = Peering(as1=currAS, as2=neighAS, origin=Peering.DIRECT) |
|
|
|
|
peering.save() |
|
|
|
|
|
|
|
|
|
# c) look for router/peering pairs |
|
|
|
|
if peer["BGP"]["neighbor_id"]: |
|
|
|
|
try: |
|
|
|
|
neighRouter = BorderRouter.objects.get(AS=neighAS, routerID=peer["BGP"]["neighbor_id"]) |
|
|
|
|
except BorderRouter.DoesNotExist: |
|
|
|
|
neighRouter = BorderRouter(AS=neighAS, routerID=peer["BGP"]["neighbor_id"], pingable=False, reachable=False) |
|
|
|
|
neighRouter.save() |
|
|
|
|
try: |
|
|
|
|
BorderRouterPair.getPairing(peering, currRouter, neighRouter) |
|
|
|
|
except BorderRouterPair.DoesNotExist: |
|
|
|
|
pairs = BorderRouterPair.objects.filter(Q(peering=peering) & (Q(router1=neighRouter, router2=None)|Q(router1=None, router2=neighRouter))) |
|
|
|
|
if pairs.count() > 0: |
|
|
|
|
pair = pairs[0] |
|
|
|
|
if pair.router1 == None: |
|
|
|
|
pair.router1 = currRouter |
|
|
|
|
else: |
|
|
|
|
pair.router2 = currRouter |
|
|
|
|
pair.save() |
|
|
|
|
else: |
|
|
|
|
pair = BorderRouterPair(peering=peering, router1=currRouter, router2=neighRouter) |
|
|
|
|
pair.save() |
|
|
|
|
|
|
|
|
|
print(" --> Announcements") |
|
|
|
|
if "routes" in data and data["routes"]: |
|
|
|
|
for route in data["routes"]: |
|
|
|
|
print(" ---->", route["prefix"]) |
|
|
|
|
if "/" not in route["prefix"]: |
|
|
|
|
continue |
|
|
|
|
|
|
|
|
|
crawlAS = currAS |
|
|
|
|
if len(route["path"]) > 0: |
|
|
|
|
crawlAS = getOrCreateAS(crawl, route["path"][0]) |
|
|
|
|
originAS = getOrCreateAS(crawl, route["path"][-1]) |
|
|
|
|
ip, prefix = route["prefix"].split("/") |
|
|
|
|
a = Announcement(router=currRouter, ip=ip, prefix=prefix, |
|
|
|
|
ASPath=" ".join(route["path"]), nextHop=route["nexthop"], |
|
|
|
|
crawlAS=crawlAS, originAS=originAS) |
|
|
|
|
a.save() |
|
|
|
|
else: |
|
|
|
|
print(" !! No routes found in host output") |
|
|
|
|
CrawlLog.log(crawl, "No routes found in host output (no bgp feed included?)", host=host, severity=CrawlLog.WARN) |
|
|
|
|
|
|
|
|
|
# 3. calculate missing data |
|
|
|
|
print(" -- Adding extra data from announcements...") |
|
|
|
|
# 3.1. use announcement data to find hidden peerings |
|
|
|
|
for announcement in Announcement.objects.filter(router__AS__crawl=crawl): |
|
|
|
|
path = announcement.ASPath.split(" ") |
|
|
|
|
if len(path) > 1: |
|
|
|
|
firstASno = path.pop(0) |
|
|
|
|
firstAS = getOrCreateAS(crawl, firstASno) |
|
|
|
|
while len(path) > 0: |
|
|
|
|
secondASno = path.pop(0) |
|
|
|
|
secondAS = getOrCreateAS(crawl, secondASno) |
|
|
|
|
|
|
|
|
|
try: |
|
|
|
|
Peering.getPeering(firstAS, secondAS) |
|
|
|
|
except Peering.DoesNotExist: |
|
|
|
|
peering = Peering(as1=firstAS, as2=secondAS, origin=Peering.PATH) |
|
|
|
|
peering.save() |
|
|
|
|
|
|
|
|
|
firstAS = secondAS |
|
|
|
|
|
|
|
|
|
# 3.2 add ASses, routers and peerings from old crawlruns (last should suffice) |
|
|
|
|
# find |
|
|
|
|
print(" --> copy old ASses") |
|
|
|
|
timerangeStart = crawl.startTime - datetime.timedelta(LAST_SEEN_DAYS) |
|
|
|
|
oldASses = AS.objects.filter(online=True, crawl__startTime__gte=timerangeStart).values("number").annotate(lastSeen=Max('crawl_id')).filter(~Q(lastSeen=crawl.pk)) |
|
|
|
|
|
|
|
|
|
# 3.2.1. copy old asses |
|
|
|
|
print(" ----> create ASses") |
|
|
|
|
for oldASdata in oldASses: |
|
|
|
|
print(" ------> AS", oldASdata["number"]) |
|
|
|
|
oldAS = AS.objects.get(number=oldASdata["number"], crawl=oldASdata["lastSeen"]) |
|
|
|
|
|
|
|
|
|
try: |
|
|
|
|
newAS = AS.objects.get(number=oldAS.number, crawl=crawl) |
|
|
|
|
if not newAS.online and not newAS.lastSeen: |
|
|
|
|
newAS.lastSeen = oldAS.crawl |
|
|
|
|
newAS.save() |
|
|
|
|
except: |
|
|
|
|
newAS = AS(number=oldAS.number, crawl=crawl, lastSeen=oldAS.crawl, directlyCrawled=False, online=False) |
|
|
|
|
newAS.save() |
|
|
|
|
|
|
|
|
|
# 3.2.2 copy peerings between old asses |
|
|
|
|
print(" ----> copy peerings") |
|
|
|
|
for oldASdata in oldASses: |
|
|
|
|
print(" ------> AS", oldASdata["number"]) |
|
|
|
|
oldAS = AS.objects.get(number=oldASdata["number"], crawl=oldASdata["lastSeen"]) |
|
|
|
|
for peering in oldAS.getPeerings(): |
|
|
|
|
print(" --------> Peering %s <--> %s" % (peering.as1.number, peering.as2.number)) |
|
|
|
|
peering = Peering( |
|
|
|
|
as1=AS.objects.get(number=peering.as1.number, crawl=crawl), |
|
|
|
|
as2=AS.objects.get(number=peering.as2.number, crawl=crawl), |
|
|
|
|
origin=peering.origin) |
|
|
|
|
peering.save() |
|
|
|
|
|
|
|
|
|
# 3.3 FIXME: do we also want to have old peerings which do not exist anymore? |
|
|
|
|
|
|
|
|
|
# 4. end crawl run |
|
|
|
|
crawl.endTime = timezone.now() |
|
|
|
|
crawl.save() |
|
|
|
|
|
|
|
|
|
# additional data |
|
|
|
|
crawl.asCount = crawl.as_set.count() |
|
|
|
|
crawl.asOnlineCount = crawl.as_set.filter(online=True).count() |
|
|
|
|
crawl.asOfflineCount = crawl.as_set.filter(online=False).count() |
|
|
|
|
crawl.peeringCount = Peering.objects.filter(Q(as1__crawl=crawl)|Q(as2__crawl=crawl)).count() |
|
|
|
|
crawl.save() |
|
|
|
|
|
|
|
|
|
print(" !! Done") |
|
|
|
|
CrawlLog.log(crawl, "Crawl completed", severity=CrawlLog.INFO) |
|
|
|
|
|
|
|
|
|
if __name__ == '__main__': |
|
|
|
|
main() |
|
|
|
|
main() |
|
|
|
|