location-import.in: optimise regular expression for filtering ASN allocations to...

[location/libloc.git] / src / python / importer.py

diff --git a/src/python/importer.py b/src/python/importer.py
index f19db4bbd28f68e95485aad1ad627f9e7091367f..4c8406c1c242aac9e1efad221e01f30e338bf89c 100644 (file)
--- a/src/python/importer.py
+++ b/src/python/importer.py
@@ -25,50 +25,68 @@ import urllib.request
 log = logging.getLogger("location.importer")
 log.propagate = 1
 
-WHOIS_SOURCES = (
+WHOIS_SOURCES = {
        # African Network Information Centre
-       "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
+       "AFRINIC": [
+               "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz"
+               ],
 
        # Asia Pacific Network Information Centre
-       "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
-       "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
-       #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
-       #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
-       "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
-       "https://ftp.apnic.net/apnic/whois/apnic.db.organisation.gz",
+       "APNIC": [
+               "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
+               "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
+               #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
+               #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
+               "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
+               "https://ftp.apnic.net/apnic/whois/apnic.db.organisation.gz"
+               ],
 
        # American Registry for Internet Numbers
        # XXX there is nothing useful for us in here
-       #"https://ftp.arin.net/pub/rr/arin.db",
+       # ARIN: [
+       #       "https://ftp.arin.net/pub/rr/arin.db"
+       # ],
 
        # Latin America and Caribbean Network Information Centre
        # XXX ???
 
        # Réseaux IP Européens
-       "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
-       "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
-       #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
-       #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
-       "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
-       "https://ftp.ripe.net/ripe/dbase/split/ripe.db.organisation.gz",
-)
-
-EXTENDED_SOURCES = (
+       "RIPE": [
+               "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
+               "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
+               #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
+               #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
+               "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
+               "https://ftp.ripe.net/ripe/dbase/split/ripe.db.organisation.gz"
+               ],
+}
+
+EXTENDED_SOURCES = {
        # African Network Information Centre
-       #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
+       # "ARIN": [
+       #       "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest"
+       # ],
 
        # Asia Pacific Network Information Centre
-       #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
+       # "APNIC": [
+       #       "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest"
+       # ],
 
        # American Registry for Internet Numbers
-       "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
+       "ARIN": [
+               "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest"
+               ],
 
        # Latin America and Caribbean Network Information Centre
-       "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
+       "LACNIC": [
+               "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest"
+               ],
 
        # Réseaux IP Européens
-       #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
-)
+       # "RIPE": [
+       #       "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest"
+       # ],
+}
 
 class Downloader(object):
        def __init__(self):