Add protobuf as dependency

Some crawls don't contain all ASNs. As we only want to convert the crawl
and not add additional information we just skip adding stub info for
non-existant ASNs.

backend/cmk_parser.py

@@ -45,7 +45,7 @@ def parseBGPData(raw, asno):
 
     # mkify
     raw = d.group(2).split("\n")
-    arr = filter(lambda _z: _z, map(lambda _y: filter(lambda _x: _x, re.split(r"\s+", _y)), raw))
+    arr = list(filter(lambda _z: _z, map(lambda _y: list(filter(lambda _x: _x, re.split(r"\s+", _y))), raw)))
 
     # parse for bird/quagga
     result = None
@@ -80,7 +80,7 @@ def parseBird(data, raw, asno):
     status = _birdFindTable(data, "show status")
     if status[2][0] != "1011-Router":
         err("Couldn't find router id in bird output")
-    peers = filter(lambda _x: _x["type"] == "BGP", _birdMakeProtocols(data))
+    peers = list(filter(lambda _x: _x["type"] == "BGP", _birdMakeProtocols(data)))
 
     if asno == None:
         err("Host is bird")
@@ -135,7 +135,7 @@ def _birdMakeProtocols(info):
     #   state (established, active)
     #   if error, last error is avilable
     protocols = []
-    for proto, data in _birdFindProtocols(info).iteritems():
+    for proto, data in _birdFindProtocols(info).items():
         protoInfo = {
             "name": proto,
             "type": data[0][1],
@@ -343,7 +343,7 @@ def _quaggaFindRoutes(raw):
                         d["network"] = lastIP
 
                     # "parse" path (everything after 61 chars, but no i)
-                    path = filter(lambda _x: _x not in ('', 'i'), line[61:].split(" "))
+                    path = list(filter(lambda _x: _x not in ('', 'i'), line[61:].split(" ")))
 
                     # currently skip incomplete routes
                     if '?' not in path:

backend/crawler.py

@@ -199,7 +199,7 @@ def convert_crawl(crawl):
             if path:
                 net.nodes[path[-1]]['prefixes'].add(prefix)
                 for path_asn in path:
-                    if path_asn not in (path[-1], path[0]):
+                    if path_asn in net.nodes and path_asn not in (path[-1], path[0]):
                         net.nodes[path_asn]['stub'] = False
 
     # add neighbor count

bgpdata/templates/bgpdata/new_new_map.html

@@ -180,13 +180,22 @@ d3.json("/api/v2/crawlrun/" + elem + "/?with_graph").then((data) => {
         .attr("rx", d => d.stub ? 34 : (40 + (d.neighbors > 5 ? 5 : 0)))
         .attr("ry", d => d.stub ? 12 : (20 + (d.neighbors > 5 ? 2 : 0)))
         .attr("fill", function(d) {
+            //if(d.directly_crawled)
+            //    return "#94FF70";
+            //else if(!d.online)
+            //    return "#FFCCCC";
+            //else if(d.stub)
+            //    return "#3291A8"
+            //else
+            //    return "#D1FFC2";
             if(d.directly_crawled)
+				//return "#55bc32";
                 return "#94FF70";
             else if(!d.online)
                 return "#FFCCCC";
-                // return "#F0FFEB";
             else if(d.stub)
-                return "#3291A8"
+                //return "#94FF70";
+                return "#E1FFE2";
             else
                 return "#D1FFC2";
         })

requirements.txt

@@ -4,3 +4,4 @@ django-rest-framework
 django-filter
 networkx
 grpcio
+protobuf