python: Import extended WHOIS data

[people/ms/libloc.git] / src / python / importer.py

diff --git a/src/python/importer.py b/src/python/importer.py
index 3abbdc5852b6e2f1b2d5259bc76b053f466ebe1a..1321e0fb88520dcc758749210681be307a5ef1bf 100644 (file)
--- a/src/python/importer.py
+++ b/src/python/importer.py
@@ -30,10 +30,10 @@ WHOIS_SOURCES = (
        "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
 
        # Asia Pacific Network Information Centre
-       "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
-       "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
-       "https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
-       "https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
+       #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
+       #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
+       #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
+       #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
        "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
        "https://ftp.apnic.net/apnic/whois/apnic.db.organisation.gz",
 
@@ -44,14 +44,31 @@ WHOIS_SOURCES = (
        # XXX ???
 
        # Réseaux IP Européens
-       "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
-       "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
-       "https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
-       "https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
+       #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
+       #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
+       #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
+       #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
        "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
        "https://ftp.ripe.net/ripe/dbase/split/ripe.db.organisation.gz",
 )
 
+EXTENDED_SOURCES = (
+       # African Network Information Centre
+       "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
+
+       # Asia Pacific Network Information Centre
+       "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
+
+       # American Registry for Internet Numbers
+       "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
+
+       # Latin America and Caribbean Network Information Centre
+       "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
+
+       # Réseaux IP Européens
+       "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
+)
+
 class Downloader(object):
        def __init__(self):
                self.proxy = None
@@ -107,20 +124,7 @@ class DownloaderContext(object):
                if self.return_blocks:
                        return iterate_over_blocks(self.body)
 
-               # Store body
-               #body = self.body
-
-               #while True:
-               #       line = body.readline()
-               #       if not line:
-               #               break
-
-               #       # Decode the line
-               #       print(line)
-               #       line = line.decode()
-
-               #       # Strip the ending
-               #       yield line.rstrip()
+               return iterate_over_lines(self.body)
 
        @property
        def headers(self):
@@ -188,3 +192,12 @@ def iterate_over_blocks(f, charsets=("utf-8", "latin1")):
 
                # Reset the block
                block = []
+
+
+def iterate_over_lines(f):
+       for line in f:
+               # Decode the line
+               line = line.decode()
+
+               # Strip the ending
+               yield line.rstrip()