#!/usr/bin/env python # This file is part of dnmapper, an AS--level mapping tool # Licensed under GNU General Public License v3 or later # Written by Sebastian Lohff (seba@someserver.de) from __future__ import print_function # prepare environment import sys sys.path.append("..") import os os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dnmapper.settings") import django django.setup() import socket import datetime from django.utils import timezone from django.db.models import Q, Max from bgpdata.models import ConfigHost, CrawlRun, CrawlLog, AS, BorderRouter, Announcement, Peering, BorderRouterPair from backend import gobgp, cmk_parser # config LAST_SEEN_DAYS = 7 def getOrCreateAS(crawl, number, online=True): currAS = None try: currAS = AS.objects.get(crawl=crawl, number=number) if online: currAS.setOnline() except AS.DoesNotExist: currAS = AS(crawl=crawl, number=number, online=online) currAS.save() return currAS def main(): # 1. create crawl run crawl = CrawlRun() crawl.startTime = timezone.now() crawl.save() CrawlLog.log(crawl, "Starting crawl run!", severity=CrawlLog.INFO) # 2. get data from all hosts, put it in the database for host in ConfigHost.objects.all(): crawl.hostsCrawled.add(host) data = None print(" -- Getting data for host %s" % host) try: if host.checkMethod == 'CMK': data = [cmk_parser.getBGPData(host.ip, host.number)] elif host.checkMethod == 'GOBGP': data = gobgp.get_bgp_data(host.ip) else: CrawlLog.log(crawl, "Method %s is not currently supported, skipping host" % host.checkMethod, host=host, severity=CrawlLog.ERROR) continue except cmk_parser.RouterParserException as e: msg = "Could not parse data for host: %s" % str(e) print("%s: %s" % (host, msg)) CrawlLog.log(crawl, msg, host=host, severity=CrawlLog.ERROR) continue except socket.error as e: msg = "Could not reach host: %s" % (e,) print("%s: %s" % (host, msg)) CrawlLog.log(crawl, msg, host=host, severity=CrawlLog.ERROR) continue for elem in data: parseData(crawl, host, elem) # 3. calculate missing data print(" -- Adding extra data from announcements...") # 3.1. use announcement data to find hidden peerings for announcement in Announcement.objects.filter(router__AS__crawl=crawl): path = announcement.ASPath.split(" ") if len(path) > 1: firstASno = path.pop(0) firstAS = getOrCreateAS(crawl, firstASno) while len(path) > 0: secondASno = path.pop(0) secondAS = getOrCreateAS(crawl, secondASno) try: Peering.getPeering(firstAS, secondAS) except Peering.DoesNotExist: peering = Peering(as1=firstAS, as2=secondAS, origin=Peering.PATH) peering.save() firstAS = secondAS # 3.2 add ASses, routers and peerings from old crawlruns (last should suffice) # find print(" --> copy old ASses") timerangeStart = crawl.startTime - datetime.timedelta(LAST_SEEN_DAYS) oldASses = AS.objects.filter(online=True, crawl__startTime__gte=timerangeStart) \ .values("number").annotate(lastSeen=Max('crawl_id')).filter(~Q(lastSeen=crawl.pk)) # 3.2.1. copy old asses print(" ----> create ASses") for oldASdata in oldASses: print(" ------> AS", oldASdata["number"]) oldAS = AS.objects.get(number=oldASdata["number"], crawl=oldASdata["lastSeen"]) try: newAS = AS.objects.get(number=oldAS.number, crawl=crawl) if not newAS.online and not newAS.lastSeen: newAS.lastSeen = oldAS.crawl newAS.save() except Exception: newAS = AS(number=oldAS.number, crawl=crawl, lastSeen=oldAS.crawl, directlyCrawled=False, online=False) newAS.save() # 3.2.2 copy peerings between old asses print(" ----> copy peerings") for oldASdata in oldASses: print(" ------> AS", oldASdata["number"]) oldAS = AS.objects.get(number=oldASdata["number"], crawl=oldASdata["lastSeen"]) for peering in oldAS.getPeerings(): print(" --------> Peering %s <--> %s" % (peering.as1.number, peering.as2.number)) peering = Peering( as1=AS.objects.get(number=peering.as1.number, crawl=crawl), as2=AS.objects.get(number=peering.as2.number, crawl=crawl), origin=peering.origin) peering.save() # 3.3 FIXME: do we also want to have old peerings which do not exist anymore? # 4. end crawl run crawl.endTime = timezone.now() crawl.save() # additional data crawl.asCount = crawl.as_set.count() crawl.asOnlineCount = crawl.as_set.filter(online=True).count() crawl.asOfflineCount = crawl.as_set.filter(online=False).count() crawl.peeringCount = Peering.objects.filter(Q(as1__crawl=crawl) | Q(as2__crawl=crawl)).count() crawl.save() print(" !! Done") CrawlLog.log(crawl, "Crawl completed", severity=CrawlLog.INFO) def parseData(crawl, host, data): print(" -- parsing...") currASno = int(data["local_as"]) currAS = getOrCreateAS(crawl, currASno) currAS.directlyCrawled = True currAS.save() currRouter = None try: currRouter = BorderRouter.objects.get(AS=currAS, routerID=data["local_id"]) currRouter.pingable = True currRouter.reachable = True currRouter.save() except BorderRouter.DoesNotExist: currRouter = BorderRouter(AS=currAS, routerID=data["local_id"], pingable=True, reachable=True) currRouter.save() print(" --> peers") for peer in data["peers"]: # peerings # data: BGP{state, neighbor_id, neighbor_as}, description # a) find/create neighbor print(" ----> Peer:", int(peer["BGP"]["neighbor_as"])) neighAS = getOrCreateAS(crawl, int(peer["BGP"]["neighbor_as"]), online=peer["BGP"]["online"]) # b) find out if a peering already exists (maybe where we only need to add our router id?) peering = None try: peering = Peering.getPeering(currAS, neighAS) except Peering.DoesNotExist: peering = Peering(as1=currAS, as2=neighAS, origin=Peering.DIRECT) peering.save() # c) look for router/peering pairs if peer["BGP"]["neighbor_id"]: try: neighRouter = BorderRouter.objects.get(AS=neighAS, routerID=peer["BGP"]["neighbor_id"]) except BorderRouter.DoesNotExist: neighRouter = BorderRouter(AS=neighAS, routerID=peer["BGP"]["neighbor_id"], pingable=False, reachable=False) neighRouter.save() try: BorderRouterPair.getPairing(peering, currRouter, neighRouter) except BorderRouterPair.DoesNotExist: pairs = BorderRouterPair.objects.filter(Q(peering=peering) & (Q(router1=neighRouter, router2=None) | Q(router1=None, router2=neighRouter))) if pairs.count() > 0: pair = pairs[0] if pair.router1 is None: pair.router1 = currRouter else: pair.router2 = currRouter pair.save() else: pair = BorderRouterPair(peering=peering, router1=currRouter, router2=neighRouter) pair.save() print(" --> Announcements") if "routes" in data and data["routes"]: for route in data["routes"]: print(" ---->", route["prefix"]) if "/" not in route["prefix"]: continue crawlAS = currAS if len(route["path"]) > 0: route["path"] = list(map(str, route["path"])) crawlAS = getOrCreateAS(crawl, route["path"][0]) originAS = getOrCreateAS(crawl, route["path"][-1]) ip, prefix = route["prefix"].split("/") a = Announcement(router=currRouter, ip=ip, prefix=prefix, ASPath=" ".join(route["path"]), nextHop=route["nexthop"], crawlAS=crawlAS, originAS=originAS) a.save() else: print(" !! No routes found in host output") CrawlLog.log(crawl, "No routes found in host output (no bgp feed included?)", host=host, severity=CrawlLog.WARN) if __name__ == '__main__': main()