seba
/
dnmapper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add stubs, neighbors, etc to crawl data 

* mark networks as stubs that are never seen in the middle of an aspath
 * add number of neighbors to each node
 * fix bug where route was an unbound variable
This commit is contained in:
Sebastian Lohff 2020-06-10 03:09:49 +02:00
parent 99c02a84d6
commit f898cd8d40
2 changed files with 12 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
backend/crawler.py
View File

@ -53,6 +53,10 @@ def crawl():
asneigh = ASLastSeenNeighbor(asn=neigh, neighbor=asls) asneigh = ASLastSeenNeighbor(asn=neigh, neighbor=asls)
asneigh.save() asneigh.save()
db_neighs.add(neigh) db_neighs.add(neigh)
crawl.endTime = timezone.now()
crawl.save()
log.info("Automated crawl done") log.info("Automated crawl done")
@ -94,6 +98,10 @@ def get_current_network():
log.exception("Could not get data from host %s method %s", host, host.checkMethod) log.exception("Could not get data from host %s method %s", host, host.checkMethod)
continue continue
# add neighbor count
for node, data in net.nodes(data=True):
data["neighbors"] = len(list(net.neighbors(node)))
log.info("Adding last seen neighbor info") log.info("Adding last seen neighbor info")
for asls in ASLastSeen.objects.all(): for asls in ASLastSeen.objects.all():
if asls.asn not in net.nodes: if asls.asn not in net.nodes:
@ -136,6 +144,7 @@ def _populate_node(net, asn):
node.setdefault("routing_table", set()) node.setdefault("routing_table", set())
node.setdefault("directly_crawled", False) node.setdefault("directly_crawled", False)
node.setdefault("online", True) node.setdefault("online", True)
node.setdefault("stub", True)
return node return node
@ -162,6 +171,8 @@ def _add_data_to_net(net, data):
if as_path[n] != as_path[n + 1]: if as_path[n] != as_path[n + 1]:
if as_path[n + 1] not in net.nodes: if as_path[n + 1] not in net.nodes:
_populate_node(net, as_path[n + 1]) _populate_node(net, as_path[n + 1])
if as_path[n + 1] not in (as_path[-1], as_path[0]):
net.nodes[as_path[n + 1]]['stub'] = False
net.add_edge(as_path[n], as_path[n + 1]) net.add_edge(as_path[n], as_path[n + 1])