You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
210 lines
6.5 KiB
210 lines
6.5 KiB
import json
|
|
import logging
|
|
import socket
|
|
import time
|
|
|
|
from django.utils import timezone
|
|
import networkx as nx
|
|
|
|
from backend import gobgp, cmk_parser
|
|
from backend.exceptions import RouterParserException
|
|
from bgpdata.models import ConfigHost, ASLastSeen, ASLastSeenNeighbor, CrawlRun
|
|
|
|
log = logging.getLogger(__name__)
|
|
FORMAT = '%(asctime)-15s %(message)s'
|
|
logging.basicConfig(level=logging.INFO, format=FORMAT)
|
|
log.setLevel(logging.INFO)
|
|
|
|
|
|
def crawl():
|
|
net = get_current_network()
|
|
crawl = make_crawl_from_net(net)
|
|
crawl.save()
|
|
|
|
# handle last seen ASNs
|
|
log.info("Updating last seen info")
|
|
last_seen = {}
|
|
for asls in ASLastSeen.objects.all():
|
|
last_seen[asls.asn] = asls
|
|
|
|
for asn, node in net.nodes(data=True):
|
|
if asn not in last_seen:
|
|
last_seen[asn] = ASLastSeen(asn=asn)
|
|
asls = last_seen[asn]
|
|
asls.online = node['online']
|
|
if node['online']:
|
|
asls.directlyCrawled = node['directly_crawled']
|
|
asls.lastSeen = timezone.now()
|
|
asls.crawlLastSeen = crawl
|
|
else:
|
|
asls.directlyCrawled = False
|
|
asls.save()
|
|
|
|
if asls.online:
|
|
neighs = net.neighbors(asn)
|
|
db_neighs = set()
|
|
for db_neigh in asls.aslastseenneighbor_set.all():
|
|
if db_neigh.asn in neighs:
|
|
db_neighs.add(asls.asn)
|
|
else:
|
|
db_neigh.delete()
|
|
for neigh in neighs:
|
|
if neigh not in db_neighs:
|
|
asneigh = ASLastSeenNeighbor(asn=neigh, neighbor=asls)
|
|
asneigh.save()
|
|
db_neighs.add(neigh)
|
|
|
|
crawl.endTime = timezone.now()
|
|
crawl.save()
|
|
|
|
log.info("Automated crawl done")
|
|
|
|
|
|
def make_crawl_from_net(net):
|
|
"""Create a CrawlRun, but don't save it"""
|
|
asCount = asOnlineCount = asOfflineCount = 0
|
|
for asn, node in net.nodes(data=True):
|
|
asCount += 1
|
|
if node['online']:
|
|
asOnlineCount += 1
|
|
else:
|
|
asOfflineCount += 1
|
|
|
|
crawl = CrawlRun()
|
|
crawl.startTime = timezone.now()
|
|
crawl.graph = net_to_json(net)
|
|
crawl.asCount = asCount
|
|
crawl.asOnlineCount = asOnlineCount
|
|
crawl.asOfflineCount = asOfflineCount
|
|
crawl.peeringCount = len(net.edges)
|
|
|
|
return crawl
|
|
|
|
|
|
def get_current_network():
|
|
net = nx.Graph()
|
|
|
|
crawl_start = time.time()
|
|
log.info("Crawl run started")
|
|
for host in ConfigHost.objects.all():
|
|
try:
|
|
if host.checkMethod == 'CMK':
|
|
data = cmk_parser.getBGPData(host.ip, host.number)
|
|
_add_data_to_net(net, data)
|
|
elif host.checkMethod == 'GOBGP':
|
|
for entry in gobgp.get_bgp_data(host.ip):
|
|
_add_data_to_net(net, entry)
|
|
except (RouterParserException, socket.error):
|
|
log.exception("Could not get data from host %s method %s", host, host.checkMethod)
|
|
continue
|
|
|
|
log.info("Adding last seen neighbor info")
|
|
for asls in ASLastSeen.objects.all():
|
|
if asls.asn not in net.nodes:
|
|
if any(neigh.asn in net.nodes for neigh in asls.aslastseenneighbor_set.all()):
|
|
_populate_node(net, asls.asn)
|
|
net.nodes[asls.asn]['online'] = False
|
|
for neigh in asls.aslastseenneighbor_set.all():
|
|
if neigh.asn not in net.nodes:
|
|
_populate_node(net, neigh.asn)
|
|
net.nodes[asls.asn]['online'] = False
|
|
|
|
# add neighbor count
|
|
for node, data in net.nodes(data=True):
|
|
data["neighbors"] = len(list(net.neighbors(node)))
|
|
|
|
log.info("Crawl done in %.2fs", time.time() - crawl_start)
|
|
|
|
# add id to edges
|
|
for n, (_, _, data) in enumerate(net.edges(data=True)):
|
|
data['id'] = n
|
|
|
|
# import IPython
|
|
# IPython.embed()
|
|
|
|
return net
|
|
|
|
|
|
def net_to_json(net):
|
|
"""Dum net to json, will replace all sets from the graph"""
|
|
# replace all sets with lists for better dumpability
|
|
for node in net.nodes.values():
|
|
for key, val in node.items():
|
|
if isinstance(val, set):
|
|
node[key] = list(val)
|
|
|
|
return json.dumps(nx.readwrite.json_graph.node_link_data(net))
|
|
|
|
|
|
def _populate_node(net, asn):
|
|
net.add_node(asn)
|
|
node = net.nodes[asn]
|
|
node.setdefault("prefixes", set())
|
|
node.setdefault("router_ids", set())
|
|
node.setdefault("routing_table", set())
|
|
node.setdefault("directly_crawled", False)
|
|
node.setdefault("online", True)
|
|
node.setdefault("stub", True)
|
|
return node
|
|
|
|
|
|
def _add_data_to_net(net, data):
|
|
asn = data['local_as']
|
|
as_node = _populate_node(net, asn)
|
|
as_node['router_ids'].add(data['local_id'])
|
|
as_node['directly_crawled'] = True
|
|
|
|
for peer in data['peers']:
|
|
pass
|
|
|
|
for route in data['routes']:
|
|
as_node['routing_table'].add((route['prefix'], tuple(route['path'])))
|
|
|
|
as_path = route['path']
|
|
if not as_path:
|
|
continue
|
|
|
|
orig_node = _populate_node(net, as_path[-1])
|
|
orig_node['prefixes'].add(route['prefix'])
|
|
|
|
for n in range(len(as_path) - 1):
|
|
if as_path[n] != as_path[n + 1]:
|
|
if as_path[n + 1] not in net.nodes:
|
|
_populate_node(net, as_path[n + 1])
|
|
if as_path[n + 1] not in (as_path[-1], as_path[0]):
|
|
net.nodes[as_path[n + 1]]['stub'] = False
|
|
net.add_edge(as_path[n], as_path[n + 1])
|
|
|
|
|
|
def convert_crawl(crawl):
|
|
net = nx.Graph()
|
|
|
|
for asn in crawl.as_set.all():
|
|
if asn.number not in net.nodes:
|
|
_populate_node(net, asn.number)
|
|
d = net.nodes[asn.number]
|
|
d['online'] = asn.online
|
|
d['directly_crawled'] = asn.directlyCrawled
|
|
for br in asn.borderrouter_set.all():
|
|
d['router_ids'].add(br.routerID)
|
|
|
|
for asn in crawl.as_set.all():
|
|
for peering in asn.getPeerings():
|
|
net.add_edge(peering.as1.number, peering.as2.number)
|
|
|
|
for ann in asn.announcement_set.all():
|
|
prefix = "{}/{}".format(ann.ip, ann.prefix)
|
|
path = list(map(int, ann.ASPath.split()))
|
|
net.nodes[asn.number]['routing_table'].add((prefix, tuple(path)))
|
|
if path:
|
|
net.nodes[path[-1]]['prefixes'].add(prefix)
|
|
for path_asn in path:
|
|
if path_asn not in (path[-1], path[0]):
|
|
net.nodes[path_asn]['stub'] = False
|
|
|
|
# add neighbor count
|
|
for node, data in net.nodes(data=True):
|
|
data["neighbors"] = len(list(net.neighbors(node)))
|
|
|
|
return net
|