From ecec36aa3df572a4247dbf7a05541de342858e88 Mon Sep 17 00:00:00 2001 From: Sebastian Lohff Date: Mon, 27 Mar 2017 00:33:29 +0200 Subject: [PATCH] Importing data --- bin/import-data | 111 ++++++++++++++++++ templates/whoisdb/handle_table_row.html | 2 +- whoisdb/forms.py | 12 +- whoisdb/migrations/0014_inetnum_origin_as.py | 20 ++++ whoisdb/migrations/0015_auto_20170326_2220.py | 19 +++ whoisdb/models.py | 6 + 6 files changed, 168 insertions(+), 2 deletions(-) create mode 100755 bin/import-data create mode 100644 whoisdb/migrations/0014_inetnum_origin_as.py create mode 100644 whoisdb/migrations/0015_auto_20170326_2220.py diff --git a/bin/import-data b/bin/import-data new file mode 100755 index 0000000..efe6960 --- /dev/null +++ b/bin/import-data @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +from __future__ import print_function + +import argparse +import json + +import sys +import re +import os +import django +import ipaddress + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dnmgmt.settings") +sys.path.append("..") +django.setup() + + +from whoisdb.models import ASBlock, ASNumber, Contact, Maintainer, InetNum + +__VERSION__ = '0.1' + +def _parser(): + parser = argparse.ArgumentParser( + #prog='foo', + #description='do some awesome foo', + ) + + #parser.add_argument("-p", "--port", default=2323, type=int, help="Your port") + #parser.add_argument("-v", "--verbose", default=False, action="store_true", help="Be more verbose") + parser.add_argument("-j", "--json", type=argparse.FileType('r'), required=True, help="Path to json file") + + parser.add_argument("--version", action="version", version="%(prog)s " + __VERSION__) + + return parser + +def main(): + parser = _parser() + args = parser.parse_args() + + defContact = Contact.objects.get(handle="DURC1-DN") + defContact + defMnt = Maintainer.objects.get(handle="DARK1-MNT") + blocks = [ + ASBlock.objects.get(handle="TRA1-ASB"), + ASBlock.objects.get(handle="UAB1-ASB") + ] + ASNumber + + + data = json.load(args.json) + print(data.keys()) + for k, v in data["handles"].items(): + print(k, v["person"]) + for asn in data["as"]: + print(asn) + if ASNumber.objects.filter(number=asn["number"]).count() > 0: + continue + + obj = ASNumber(number=asn["number"]) + block = None + for block in blocks: + if obj.number >= block.asBegin and obj.number <= block.asEnd: + obj.asblock = block + break + else: + raise ValueError("AS %d does not fit a block" % asn["number"]) + + if len(asn['admin_c']) >= 1: + name = data["handles"][asn["admin_c"][0]]["person"] + obj.name = "Imported AS from %s" % name + obj.handle = ASNumber.genGenericHandle(re.sub("[^a-zA-Z0-9 ]", "", name)) + else: + obj.name = "Imported AS without admin info" + obj.handle = ASNumber.genGenericHandle("Unknown Imported AS") + obj.description = "Object has been imported from old DB and has not yet been edited" + obj.save() + obj.mnt_by.add(defMnt) + obj.save() + + for net in data["network"]: + print(net) + network = ipaddress.ip_network(net["prefix"]) + + if InetNum.objects.filter(address=str(network.network_address), netmask=network.prefixlen).count() > 0: + continue + + origin = None + if net["origin"]: + origin = ASNumber.objects.get(number=net["origin"]) + obj = InetNum(address=str(network.network_address), netmask=network.prefixlen) + obj.protocol = InetNum.IPv4 if network.version == 4 else InetNum.IPv6 + + x = list(filter(lambda _x: _x['number'] == net["origin"], data["as"])) + if len(x) > 0 and x[0]["admin_c"]: + name = data["handles"][x[0]["admin_c"][0]]["person"] + obj.name = "Imported Network from %s" % name + obj.handle = InetNum.genGenericHandle(re.sub("[^a-zA-Z0-9 ]", "", name)) + else: + obj.name = "Imported Network without admin info" + obj.handle = InetNum.genGenericHandle("Unknown Imported Network") + obj.description = "Object has been imported from old DB and has not yet been edited" + obj.save() + + obj.mnt_by.add(defMnt) + if origin: + obj.origin_as.add(origin) + obj.save() + +if __name__ == '__main__': + main() diff --git a/templates/whoisdb/handle_table_row.html b/templates/whoisdb/handle_table_row.html index e2792a9..5446dab 100644 --- a/templates/whoisdb/handle_table_row.html +++ b/templates/whoisdb/handle_table_row.html @@ -6,7 +6,7 @@ {{ obj|linkObject }} {{ objType }} - {{ obj.name }}{{ obj.description }} + {{ obj.name }}{% if obj.name and obj.description %} / {% endif %}{{ obj.description }} {{ obj.getResource|default:"-" }} {{ obj.mnt_by.all|linkObjects|default:"-" }}{% if obj.mnt_lower.all %} / lower: {{ obj.mnt_lower.all|linkObjects }}{% endif %} {{ obj.admin_c.all|linkObjects }} diff --git a/whoisdb/forms.py b/whoisdb/forms.py index eed08b1..544788e 100644 --- a/whoisdb/forms.py +++ b/whoisdb/forms.py @@ -116,7 +116,7 @@ class InetNumForm(MntFormMixin, WhoisObjectFormMixin, forms.ModelForm): class Meta: model = InetNum - fields = ['handle', 'protocol', 'parent_range', 'prefix', 'name', 'description', 'mnt_by', 'mnt_lower', 'admin_c'] + fields = ['handle', 'protocol', 'parent_range', 'prefix', 'name', 'description', 'origin_as', 'mnt_by', 'mnt_lower', 'admin_c'] def __init__(self, *args, **kwargs): super(InetNumForm, self).__init__(*args, **kwargs) @@ -132,6 +132,7 @@ class InetNumForm(MntFormMixin, WhoisObjectFormMixin, forms.ModelForm): #self.fields['parent_range'].queryset = self.fields['parent_range'].queryset.distinct() self.fields['parent_range'].queryset = InetNum.getMntQueryset(mnts, self.instance, "parent_range") + self.fields['origin_as'].queryset = ASNumber.getMntQueryset(mnts, self.instance, "origin_as") def clean_prefix(self): # make sure this is a subnet we're getting @@ -148,6 +149,9 @@ class InetNumForm(MntFormMixin, WhoisObjectFormMixin, forms.ModelForm): def clean_parent_range(self): parent_range = self.cleaned_data.get('parent_range', None) + if parent_range.origin_as.count() > 0: + raise forms.ValidationError("Parent range has origin as set") + # allow parent range to be unset for already present objects if not parent_range and (self._create or not self._create and self.instance.parent_range): raise forms.ValidationError("Parent range must be set") @@ -167,9 +171,15 @@ class InetNumForm(MntFormMixin, WhoisObjectFormMixin, forms.ModelForm): if not self._editLower: if not self.errors: + if not self._create and self.cleaned_data['origin_as']: + if self.instance.inetnum_set.count() > 0: + ranges = ", ".join(map(str, self.instance.inetnum_set.all())) + raise forms.ValidationError("You cannot set an origin as if there are already existing subranges (%s)" % (ranges)) + prefix = cleaned_data['prefix'] parent = cleaned_data['parent_range'] if parent: + parentNet = parent.getNetwork() if cleaned_data['protocol'] != parent.protocol: diff --git a/whoisdb/migrations/0014_inetnum_origin_as.py b/whoisdb/migrations/0014_inetnum_origin_as.py new file mode 100644 index 0000000..84f3198 --- /dev/null +++ b/whoisdb/migrations/0014_inetnum_origin_as.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2017-03-26 03:28 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('whoisdb', '0013_auto_20170303_1206'), + ] + + operations = [ + migrations.AddField( + model_name='inetnum', + name='origin_as', + field=models.ManyToManyField(blank=True, to='whoisdb.ASNumber'), + ), + ] diff --git a/whoisdb/migrations/0015_auto_20170326_2220.py b/whoisdb/migrations/0015_auto_20170326_2220.py new file mode 100644 index 0000000..26cda79 --- /dev/null +++ b/whoisdb/migrations/0015_auto_20170326_2220.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.5 on 2017-03-26 22:20 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('whoisdb', '0014_inetnum_origin_as'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='inetnum', + unique_together=set([('address', 'netmask')]), + ), + ] diff --git a/whoisdb/models.py b/whoisdb/models.py index 56a13a4..e1ce5ef 100644 --- a/whoisdb/models.py +++ b/whoisdb/models.py @@ -222,6 +222,11 @@ class ASNumber(MntdObject): class InetNum(MntdObject): + class Meta: + unique_together = ( + ("address", "netmask"), + ) + handleSuffix = "NET" IPv4 = "ipv4" @@ -234,6 +239,7 @@ class InetNum(MntdObject): parent_range = models.ForeignKey("InetNum", models.CASCADE, null=True, blank=True, default=None) name = models.CharField(max_length=64) description = models.CharField(max_length=64, blank=True) + origin_as = models.ManyToManyField(ASNumber, blank=True) admin_c = models.ManyToManyField("Contact") mnt_lower = models.ManyToManyField(Maintainer, related_name='lower_inetnum_set', blank=True)