import json import logging import socket import time from django.utils import timezone import networkx as nx from backend import gobgp, cmk_parser from backend.exceptions import RouterParserException from bgpdata.models import ConfigHost, ASLastSeen, ASLastSeenNeighbor, CrawlRun log = logging.getLogger(__name__) FORMAT = '%(asctime)-15s %(message)s' logging.basicConfig(level=logging.INFO, format=FORMAT) log.setLevel(logging.INFO) def crawl(): net = get_current_network() crawl = make_crawl_from_net(net) crawl.save() # handle last seen ASNs log.info("Updating last seen info") last_seen = {} for asls in ASLastSeen.objects.all(): last_seen[asls.asn] = asls for asn, node in net.nodes(data=True): if asn not in last_seen: last_seen[asn] = ASLastSeen(asn=asn) asls = last_seen[asn] asls.online = node['online'] if node['online']: asls.directlyCrawled = node['directly_crawled'] asls.lastSeen = timezone.now() asls.crawlLastSeen = crawl else: asls.directlyCrawled = False asls.save() if asls.online: neighs = net.neighbors(asn) db_neighs = set() for db_neigh in asls.aslastseenneighbor_set.all(): if db_neigh.asn in neighs: db_neighs.add(asls.asn) else: db_neigh.delete() for neigh in neighs: if neigh not in db_neighs: asneigh = ASLastSeenNeighbor(asn=neigh, neighbor=asls) asneigh.save() db_neighs.add(neigh) crawl.endTime = timezone.now() crawl.save() log.info("Automated crawl done") def make_crawl_from_net(net): """Create a CrawlRun, but don't save it""" asCount = asOnlineCount = asOfflineCount = 0 for asn, node in net.nodes(data=True): asCount += 1 if node['online']: asOnlineCount += 1 else: asOfflineCount += 1 crawl = CrawlRun() crawl.startTime = timezone.now() crawl.graph = net_to_json(net) crawl.asCount = asCount crawl.asOnlineCount = asOnlineCount crawl.asOfflineCount = asOfflineCount crawl.peeringCount = len(net.edges) return crawl def get_current_network(): net = nx.Graph() crawl_start = time.time() log.info("Crawl run started") for host in ConfigHost.objects.all(): try: if host.checkMethod == 'CMK': data = cmk_parser.getBGPData(host.ip, host.number) _add_data_to_net(net, data) elif host.checkMethod == 'GOBGP': for entry in gobgp.get_bgp_data(host.ip): _add_data_to_net(net, entry) except (RouterParserException, socket.error): log.exception("Could not get data from host %s method %s", host, host.checkMethod) continue log.info("Adding last seen neighbor info") for asls in ASLastSeen.objects.all(): if asls.asn not in net.nodes: if any(neigh.asn in net.nodes for neigh in asls.aslastseenneighbor_set.all()): _populate_node(net, asls.asn) net.nodes[asls.asn]['online'] = False for neigh in asls.aslastseenneighbor_set.all(): if neigh.asn not in net.nodes: _populate_node(net, neigh.asn) net.nodes[asls.asn]['online'] = False # add neighbor count for node, data in net.nodes(data=True): data["neighbors"] = len(list(net.neighbors(node))) log.info("Crawl done in %.2fs", time.time() - crawl_start) # add id to edges for n, (_, _, data) in enumerate(net.edges(data=True)): data['id'] = n # import IPython # IPython.embed() return net def net_to_json(net): """Dum net to json, will replace all sets from the graph""" # replace all sets with lists for better dumpability for node in net.nodes.values(): for key, val in node.items(): if isinstance(val, set): node[key] = list(val) return json.dumps(nx.readwrite.json_graph.node_link_data(net)) def _populate_node(net, asn): net.add_node(asn) node = net.nodes[asn] node.setdefault("prefixes", set()) node.setdefault("router_ids", set()) node.setdefault("routing_table", set()) node.setdefault("directly_crawled", False) node.setdefault("online", True) node.setdefault("stub", True) return node def _add_data_to_net(net, data): asn = data['local_as'] as_node = _populate_node(net, asn) as_node['router_ids'].add(data['local_id']) as_node['directly_crawled'] = True for peer in data['peers']: pass for route in data['routes']: as_node['routing_table'].add((route['prefix'], tuple(route['path']))) as_path = route['path'] if not as_path: continue orig_node = _populate_node(net, as_path[-1]) orig_node['prefixes'].add(route['prefix']) for n in range(len(as_path) - 1): if as_path[n] != as_path[n + 1]: if as_path[n + 1] not in net.nodes: _populate_node(net, as_path[n + 1]) if as_path[n + 1] not in (as_path[-1], as_path[0]): net.nodes[as_path[n + 1]]['stub'] = False net.add_edge(as_path[n], as_path[n + 1]) def convert_crawl(crawl): net = nx.Graph() for asn in crawl.as_set.all(): if asn.number not in net.nodes: _populate_node(net, asn.number) d = net.nodes[asn.number] d['online'] = asn.online d['directly_crawled'] = asn.directlyCrawled for br in asn.borderrouter_set.all(): d['router_ids'].add(br.routerID) for asn in crawl.as_set.all(): for peering in asn.getPeerings(): net.add_edge(peering.as1.number, peering.as2.number) for ann in asn.announcement_set.all(): prefix = "{}/{}".format(ann.ip, ann.prefix) path = list(map(int, ann.ASPath.split())) net.nodes[asn.number]['routing_table'].add((prefix, tuple(path))) if path: net.nodes[path[-1]]['prefixes'].add(prefix) for path_asn in path: if path_asn not in (path[-1], path[0]): net.nodes[path_asn]['stub'] = False # add neighbor count for node, data in net.nodes(data=True): data["neighbors"] = len(list(net.neighbors(node))) return net