#!/usr/bin/env python2 from __future__ import print_function # prepare environment import sys sys.path.append("..") import os os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dnmapper.settings") import django django.setup() from django.utils import timezone from django.db.models import Q from bgpdata.models import ConfigHost, CrawlRun, CrawlLog, AS, BorderRouter, Announcement, Peering, BorderRouterPair from routerparsers import getBGPData, RouterParserException def getOrCreateAS(crawl, number): currAS = None try: currAS = AS.objects.get(crawl=crawl, number=number) except AS.DoesNotExist: currAS = AS(crawl=crawl, number=number) currAS.save() return currAS def main(): # 1. create crawl run crawl = CrawlRun() crawl.startTime = timezone.now() crawl.save() CrawlLog.log(crawl, "Starting crawl run!", severity=CrawlLog.INFO) # 2. get data from all hosts, put it in the database for host in ConfigHost.objects.all(): data = None print(" -- Getting data for host %s" % host) try: if host.checkMethod == 'CMK': data = getBGPData(host.ip, host.number) else: CrawlLog.log(crawl, "Method %s is not currently supported, skipping host" % host.checkMethod, host=host, severity=CrawlLog.ERROR) continue except RouterParserException as e: msg = "Could not parse data for host: %s" % str(e) print("%s: %s" % (host, msg)) CrawlLog.log(crawl, msg, host=host, severity=CrawlLog.ERROR) continue print(" -- parsing...") currASno = int(data["local_as"]) currAS = getOrCreateAS(crawl, currASno) currRouter = None try: currRouter = BorderRouter.objects.get(AS=currAS, routerID=data["local_id"]) currRouter.pingable = True currRouter.reachable = True currRouter.save() except BorderRouter.DoesNotExist: currRouter = BorderRouter(AS=currAS, routerID=data["local_id"], pingable=True, reachable=True) currRouter.save() print(" --> peers") for peer in data["peers"]: # peerings # data: BGP{state, neighbor_id, neighbor_as}, description # a) find/create neighbor print(" ----> Peer:", int(peer["BGP"]["neighbor_as"])) neighAS = None try: neighAS = AS.objects.get(crawl=crawl, number=int(peer["BGP"]["neighbor_as"])) except AS.DoesNotExist: neighAS = AS(crawl=crawl, number=int(peer["BGP"]["neighbor_as"])) neighAS.save() # b) find out if a peering already exists (maybe where we only need to add our router id?) peering = None try: peering = Peering.getPeering(currAS, neighAS) except Peering.DoesNotExist: peering = Peering(as1=currAS, as2=neighAS, origin=Peering.DIRECT) peering.save() # c) look for router/peering pairs if peer["BGP"]["neighbor_id"]: try: neighRouter = BorderRouter.objects.get(AS=neighAS, routerID=peer["BGP"]["neighbor_id"]) except BorderRouter.DoesNotExist: neighRouter = BorderRouter(AS=currAS, routerID=peer["BGP"]["neighbor_id"], pingable=False, reachable=False) neighRouter.save() try: BorderRouterPair.getPairing(peering, currRouter, neighRouter) except BorderRouterPair.DoesNotExist: pairs = BorderRouterPair.objects.filter(Q(peering=peering) & (Q(router1=neighRouter, router2=None)|Q(router1=None, router2=neighRouter))) if pairs.count() > 0: pair = pairs[0] if pair.router1 == None: pair.router1 = currRouter else: pair.router2 = currRouter pair.save() else: pair = BorderRouterPair(peering=peering, router1=currRouter, router2=neighRouter) pair.save() print(" --> Announcements") if "routes" in data and data["routes"]: for route in data["routes"]: print(" ---->", route["prefix"]) if "/" not in route["prefix"]: continue ip, prefix = route["prefix"].split("/") a = Announcement(router=currRouter, ip=ip, prefix=prefix, ASPath=" ".join(route["path"]), nextHop=route["nexthop"], originAS=currAS) a.save() else: print(" !! No routes found in host output") CrawlLog.log(crawl, "No routes found in host output (no bgp feed included?)", host=host, severity=CrawlLog.WARN) # 3. calculate missing data print(" -- Adding extra data from announcements...") # 3.1. use announcement data to find hidden peerings for announcement in Announcement.objects.filter(router__AS__crawl=crawl): path = announcement.ASPath.split(" ") if len(path) > 1: firstASno = path.pop(0) firstAS = getOrCreateAS(crawl, firstASno) while len(path) > 0: secondASno = path.pop(0) secondAS = getOrCreateAS(crawl, secondASno) try: Peering.getPeering(firstAS, secondAS) except Peering.DoesNotExist: peering = Peering(as1=firstAS, as2=secondAS, origin=Peering.PATH) peering.save() firstAS = secondAS # 4. end crawl run crawl.endTime = timezone.now() crawl.save() print(" !! Done") CrawlLog.log(crawl, "Crawl completed", severity=CrawlLog.INFO) if __name__ == '__main__': main()