src/python/location-importer.in

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # libloc - A library to determine the location of someone on the Internet     #
   5 #                                                                             #
   6 # Copyright (C) 2020 IPFire Development Team <info@ipfire.org>                #
   7 #                                                                             #
   8 # This library is free software; you can redistribute it and/or               #
   9 # modify it under the terms of the GNU Lesser General Public                  #
  10 # License as published by the Free Software Foundation; either                #
  11 # version 2.1 of the License, or (at your option) any later version.          #
  12 #                                                                             #
  13 # This library is distributed in the hope that it will be useful,             #
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU           #
  16 # Lesser General Public License for more details.                             #
  17 #                                                                             #
  18 ###############################################################################
  19
  20 import argparse
  21 import ipaddress
  22 import logging
  23 import math
  24 import re
  25 import sys
  26
  27 # Load our location module
  28 import location
  29 import location.database
  30 import location.importer
  31 from location.i18n import _
  32
  33 # Initialise logging
  34 log = logging.getLogger("location.importer")
  35 log.propagate = 1
  36
  37 INVALID_ADDRESSES = (
  38         "0.0.0.0",
  39         "::/0",
  40         "0::/0",
  41 )
  42
  43 class CLI(object):
  44         def parse_cli(self):
  45                 parser = argparse.ArgumentParser(
  46                         description=_("Location Importer Command Line Interface"),
  47                 )
  48                 subparsers = parser.add_subparsers()
  49
  50                 # Global configuration flags
  51                 parser.add_argument("--debug", action="store_true",
  52                         help=_("Enable debug output"))
  53
  54                 # version
  55                 parser.add_argument("--version", action="version",
  56                         version="%(prog)s @VERSION@")
  57
  58                 # Database
  59                 parser.add_argument("--database-host", required=True,
  60                         help=_("Database Hostname"), metavar=_("HOST"))
  61                 parser.add_argument("--database-name", required=True,
  62                         help=_("Database Name"), metavar=_("NAME"))
  63                 parser.add_argument("--database-username", required=True,
  64                         help=_("Database Username"), metavar=_("USERNAME"))
  65                 parser.add_argument("--database-password", required=True,
  66                         help=_("Database Password"), metavar=_("PASSWORD"))
  67
  68                 # Update WHOIS
  69                 update_whois = subparsers.add_parser("update-whois", help=_("Update WHOIS Information"))
  70                 update_whois.set_defaults(func=self.handle_update_whois)
  71
  72                 args = parser.parse_args()
  73
  74                 # Enable debug logging
  75                 if args.debug:
  76                         log.setLevel(logging.DEBUG)
  77
  78                 # Print usage if no action was given
  79                 if not "func" in args:
  80                         parser.print_usage()
  81                         sys.exit(2)
  82
  83                 return args
  84
  85         def run(self):
  86                 # Parse command line arguments
  87                 args = self.parse_cli()
  88
  89                 # Initialise database
  90                 self.db = self._setup_database(args)
  91
  92                 # Call function
  93                 ret = args.func(args)
  94
  95                 # Return with exit code
  96                 if ret:
  97                         sys.exit(ret)
  98
  99                 # Otherwise just exit
 100                 sys.exit(0)
 101
 102         def _setup_database(self, ns):
 103                 """
 104                         Initialise the database
 105                 """
 106                 # Connect to database
 107                 db = location.database.Connection(
 108                         host=ns.database_host, database=ns.database_name,
 109                         user=ns.database_username, password=ns.database_password,
 110                 )
 111
 112                 with db.transaction():
 113                         db.execute("""
 114                                 -- autnums
 115                                 CREATE TABLE IF NOT EXISTS autnums(number integer, name text);
 116                                 CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number);
 117
 118                                 -- networks
 119                                 CREATE TABLE IF NOT EXISTS networks(network inet, autnum integer, country text);
 120                                 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
 121                         """)
 122
 123                 return db
 124
 125         def handle_update_whois(self, ns):
 126                 downloader = location.importer.Downloader()
 127
 128                 # Download all sources
 129                 with self.db.transaction():
 130                         # Create some temporary tables to store parsed data
 131                         self.db.execute("""
 132                                 CREATE TEMPORARY TABLE _autnums(number integer, organization text)
 133                                         ON COMMIT DROP;
 134                                 CREATE UNIQUE INDEX _autnums_number ON _autnums(number);
 135
 136                                 CREATE TEMPORARY TABLE _organizations(handle text, name text)
 137                                         ON COMMIT DROP;
 138                                 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
 139                         """)
 140
 141                         for source in location.importer.WHOIS_SOURCES:
 142                                 with downloader.request(source, return_blocks=True) as f:
 143                                         for block in f:
 144                                                 self._parse_block(block)
 145
 146                         self.db.execute("""
 147                                 INSERT INTO autnums(number, name)
 148                                         SELECT _autnums.number, _organizations.name FROM _autnums
 149                                                 LEFT JOIN _organizations ON _autnums.organization = _organizations.handle
 150                                 ON CONFLICT (number) DO UPDATE SET name = excluded.name;
 151                         """)
 152
 153                 # Download all extended sources
 154                 for source in location.importer.EXTENDED_SOURCES:
 155                         with self.db.transaction():
 156                                 # Download data
 157                                 with downloader.request(source) as f:
 158                                         for line in f:
 159                                                 self._parse_line(line)
 160
 161         def _parse_block(self, block):
 162                 # Get first line to find out what type of block this is
 163                 line = block[0]
 164
 165                 # aut-num
 166                 if line.startswith("aut-num:"):
 167                         return self._parse_autnum_block(block)
 168
 169                 # organisation
 170                 elif line.startswith("organisation:"):
 171                         return self._parse_org_block(block)
 172
 173         def _parse_autnum_block(self, block):
 174                 autnum = {}
 175                 for line in block:
 176                         # Split line
 177                         key, val = split_line(line)
 178
 179                         if key == "aut-num":
 180                                 m = re.match(r"^(AS|as)(\d+)", val)
 181                                 if m:
 182                                         autnum["asn"] = m.group(2)
 183
 184                         elif key == "org":
 185                                 autnum[key] = val
 186
 187                 # Skip empty objects
 188                 if not autnum:
 189                         return
 190
 191                 # Insert into database
 192                 self.db.execute("INSERT INTO _autnums(number, organization) \
 193                         VALUES(%s, %s) ON CONFLICT (number) DO UPDATE SET \
 194                                 organization = excluded.organization",
 195                         autnum.get("asn"), autnum.get("org"),
 196                 )
 197
 198         def _parse_org_block(self, block):
 199                 org = {}
 200                 for line in block:
 201                         # Split line
 202                         key, val = split_line(line)
 203
 204                         if key in ("organisation", "org-name"):
 205                                 org[key] = val
 206
 207                 # Skip empty objects
 208                 if not org:
 209                         return
 210
 211                 self.db.execute("INSERT INTO _organizations(handle, name) \
 212                         VALUES(%s, %s) ON CONFLICT (handle) DO \
 213                         UPDATE SET name = excluded.name",
 214                         org.get("organisation"), org.get("org-name"),
 215                 )
 216
 217         def _parse_line(self, line):
 218                 # Skip version line
 219                 if line.startswith("2"):
 220                         return
 221
 222                 # Skip comments
 223                 if line.startswith("#"):
 224                         return
 225
 226                 try:
 227                         registry, country_code, type, line = line.split("|", 3)
 228                 except:
 229                         log.warning("Could not parse line: %s" % line)
 230                         return
 231
 232                 # Skip any lines that are for stats only
 233                 if country_code == "*":
 234                         return
 235
 236                 if type in ("ipv6", "ipv4"):
 237                         return self._parse_ip_line(country_code, type, line)
 238
 239         def _parse_ip_line(self, country, type, line):
 240                 try:
 241                         address, prefix, date, status, organization = line.split("|")
 242                 except ValueError:
 243                         organization = None
 244
 245                         # Try parsing the line without organization
 246                         try:
 247                                 address, prefix, date, status = line.split("|")
 248                         except ValueError:
 249                                 log.warning("Unhandled line format: %s" % line)
 250                                 return
 251
 252                 # Skip anything that isn't properly assigned
 253                 if not status in ("assigned", "allocated"):
 254                         return
 255
 256                 # Cast prefix into an integer
 257                 try:
 258                         prefix = int(prefix)
 259                 except:
 260                         log.warning("Invalid prefix: %s" % prefix)
 261
 262                 # Fix prefix length for IPv4
 263                 if type == "ipv4":
 264                         prefix = 32 - int(math.log(prefix, 2))
 265
 266                 # Try to parse the address
 267                 try:
 268                         network = ipaddress.ip_network("%s/%s" % (address, prefix), strict=False)
 269                 except ValueError:
 270                         log.warning("Invalid IP address: %s" % address)
 271                         return
 272
 273                 self.db.execute("INSERT INTO networks(network, country) \
 274                         VALUES(%s, %s) ON CONFLICT (network) DO \
 275                         UPDATE SET country = excluded.country",
 276                         "%s" % network, country,
 277                 )
 278
 279
 280 def split_line(line):
 281         key, colon, val = line.partition(":")
 282
 283         # Strip any excess space
 284         key = key.strip()
 285         val = val.strip()
 286
 287         return key, val
 288
 289 def main():
 290         # Run the command line interface
 291         c = CLI()
 292         c.run()
 293
 294 main()