location: Provide a return code if the database does not need to be updated.
[people/ms/libloc.git] / src / python / location-importer.in
1 #!/usr/bin/python3
2 ###############################################################################
3 #                                                                             #
4 # libloc - A library to determine the location of someone on the Internet     #
5 #                                                                             #
6 # Copyright (C) 2020 IPFire Development Team <info@ipfire.org>                #
7 #                                                                             #
8 # This library is free software; you can redistribute it and/or               #
9 # modify it under the terms of the GNU Lesser General Public                  #
10 # License as published by the Free Software Foundation; either                #
11 # version 2.1 of the License, or (at your option) any later version.          #
12 #                                                                             #
13 # This library is distributed in the hope that it will be useful,             #
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU           #
16 # Lesser General Public License for more details.                             #
17 #                                                                             #
18 ###############################################################################
19
20 import argparse
21 import ipaddress
22 import logging
23 import math
24 import re
25 import sys
26 import telnetlib
27
28 # Load our location module
29 import location
30 import location.database
31 import location.importer
32 from location.i18n import _
33
34 # Initialise logging
35 log = logging.getLogger("location.importer")
36 log.propagate = 1
37
38 class CLI(object):
39         def parse_cli(self):
40                 parser = argparse.ArgumentParser(
41                         description=_("Location Importer Command Line Interface"),
42                 )
43                 subparsers = parser.add_subparsers()
44
45                 # Global configuration flags
46                 parser.add_argument("--debug", action="store_true",
47                         help=_("Enable debug output"))
48                 parser.add_argument("--quiet", action="store_true",
49                         help=_("Enable quiet mode"))
50
51                 # version
52                 parser.add_argument("--version", action="version",
53                         version="%(prog)s @VERSION@")
54
55                 # Database
56                 parser.add_argument("--database-host", required=True,
57                         help=_("Database Hostname"), metavar=_("HOST"))
58                 parser.add_argument("--database-name", required=True,
59                         help=_("Database Name"), metavar=_("NAME"))
60                 parser.add_argument("--database-username", required=True,
61                         help=_("Database Username"), metavar=_("USERNAME"))
62                 parser.add_argument("--database-password", required=True,
63                         help=_("Database Password"), metavar=_("PASSWORD"))
64
65                 # Write Database
66                 write = subparsers.add_parser("write", help=_("Write database to file"))
67                 write.set_defaults(func=self.handle_write)
68                 write.add_argument("file", nargs=1, help=_("Database File"))
69                 write.add_argument("--signing-key", nargs="?", type=open, help=_("Signing Key"))
70                 write.add_argument("--backup-signing-key", nargs="?", type=open, help=_("Backup Signing Key"))
71                 write.add_argument("--vendor", nargs="?", help=_("Sets the vendor"))
72                 write.add_argument("--description", nargs="?", help=_("Sets a description"))
73                 write.add_argument("--license", nargs="?", help=_("Sets the license"))
74                 write.add_argument("--version", type=int, help=_("Database Format Version"))
75
76                 # Update WHOIS
77                 update_whois = subparsers.add_parser("update-whois", help=_("Update WHOIS Information"))
78                 update_whois.set_defaults(func=self.handle_update_whois)
79
80                 # Update announcements
81                 update_announcements = subparsers.add_parser("update-announcements",
82                         help=_("Update BGP Annoucements"))
83                 update_announcements.set_defaults(func=self.handle_update_announcements)
84                 update_announcements.add_argument("server", nargs=1,
85                         help=_("Route Server to connect to"), metavar=_("SERVER"))
86
87                 # Update overrides
88                 update_overrides = subparsers.add_parser("update-overrides",
89                         help=_("Update overrides"),
90                 )
91                 update_overrides.add_argument(
92                         "files", nargs="+", help=_("Files to import"),
93                 )
94                 update_overrides.set_defaults(func=self.handle_update_overrides)
95
96                 # Import countries
97                 import_countries = subparsers.add_parser("import-countries",
98                         help=_("Import countries"),
99                 )
100                 import_countries.add_argument("file", nargs=1, type=argparse.FileType("r"),
101                         help=_("File to import"))
102                 import_countries.set_defaults(func=self.handle_import_countries)
103
104                 args = parser.parse_args()
105
106                 # Configure logging
107                 if args.debug:
108                         location.logger.set_level(logging.DEBUG)
109                 elif args.quiet:
110                         location.logger.set_level(logging.WARNING)
111
112                 # Print usage if no action was given
113                 if not "func" in args:
114                         parser.print_usage()
115                         sys.exit(2)
116
117                 return args
118
119         def run(self):
120                 # Parse command line arguments
121                 args = self.parse_cli()
122
123                 # Initialise database
124                 self.db = self._setup_database(args)
125
126                 # Call function
127                 ret = args.func(args)
128
129                 # Return with exit code
130                 if ret:
131                         sys.exit(ret)
132
133                 # Otherwise just exit
134                 sys.exit(0)
135
136         def _setup_database(self, ns):
137                 """
138                         Initialise the database
139                 """
140                 # Connect to database
141                 db = location.database.Connection(
142                         host=ns.database_host, database=ns.database_name,
143                         user=ns.database_username, password=ns.database_password,
144                 )
145
146                 with db.transaction():
147                         db.execute("""
148                                 -- announcements
149                                 CREATE TABLE IF NOT EXISTS announcements(network inet, autnum bigint,
150                                         first_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP,
151                                         last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
152                                 CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
153                                 CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
154
155                                 -- autnums
156                                 CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
157                                 CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number);
158
159                                 -- countries
160                                 CREATE TABLE IF NOT EXISTS countries(
161                                         country_code text NOT NULL, name text NOT NULL, continent_code text NOT NULL);
162                                 CREATE UNIQUE INDEX IF NOT EXISTS countries_country_code ON countries(country_code);
163
164                                 -- networks
165                                 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
166                                 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
167                                 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
168
169                                 -- overrides
170                                 CREATE TABLE IF NOT EXISTS autnum_overrides(
171                                         number bigint NOT NULL,
172                                         name text,
173                                         country text,
174                                         is_anonymous_proxy boolean,
175                                         is_satellite_provider boolean,
176                                         is_anycast boolean
177                                 );
178                                 CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number
179                                         ON autnum_overrides(number);
180
181                                 CREATE TABLE IF NOT EXISTS network_overrides(
182                                         network inet NOT NULL,
183                                         country text,
184                                         is_anonymous_proxy boolean,
185                                         is_satellite_provider boolean,
186                                         is_anycast boolean
187                                 );
188                                 CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
189                                         ON network_overrides(network);
190                         """)
191
192                 return db
193
194         def handle_write(self, ns):
195                 """
196                         Compiles a database in libloc format out of what is in the database
197                 """
198                 # Allocate a writer
199                 writer = location.Writer(ns.signing_key, ns.backup_signing_key)
200
201                 # Set all metadata
202                 if ns.vendor:
203                         writer.vendor = ns.vendor
204
205                 if ns.description:
206                         writer.description = ns.description
207
208                 if ns.license:
209                         writer.license = ns.license
210
211                 # Add all Autonomous Systems
212                 log.info("Writing Autonomous Systems...")
213
214                 # Select all ASes with a name
215                 rows = self.db.query("""
216                         SELECT
217                                 autnums.number AS number,
218                                 COALESCE(
219                                         (SELECT overrides.name FROM autnum_overrides overrides
220                                                 WHERE overrides.number = autnums.number),
221                                         autnums.name
222                                 ) AS name
223                                 FROM autnums
224                                 WHERE name <> %s ORDER BY number
225                         """, "")
226
227                 for row in rows:
228                         a = writer.add_as(row.number)
229                         a.name = row.name
230
231                 # Add all networks
232                 log.info("Writing networks...")
233
234                 # Select all known networks
235                 rows = self.db.query("""
236                         -- Get a (sorted) list of all known networks
237                         WITH known_networks AS (
238                                         SELECT network FROM announcements
239                                 UNION
240                                         SELECT network FROM networks
241                                 ORDER BY network
242                         )
243
244                         -- Return a list of those networks enriched with all
245                         -- other information that we store in the database
246                         SELECT
247                                 DISTINCT ON (known_networks.network)
248                                 known_networks.network AS network,
249                                 announcements.autnum AS autnum,
250
251                                 -- Country
252                                 COALESCE(
253                                         (
254                                                 SELECT country FROM network_overrides overrides
255                                                         WHERE announcements.network <<= overrides.network
256                                                         ORDER BY masklen(overrides.network) DESC
257                                                         LIMIT 1
258                                         ),
259                                         (
260                                                 SELECT country FROM autnum_overrides overrides
261                                                         WHERE announcements.autnum = overrides.number
262                                         ),
263                                         networks.country
264                                 ) AS country,
265
266                                 -- Flags
267                                 COALESCE(
268                                         (
269                                                 SELECT is_anonymous_proxy FROM network_overrides overrides
270                                                         WHERE announcements.network <<= overrides.network
271                                                         ORDER BY masklen(overrides.network) DESC
272                                                         LIMIT 1
273                                         ),
274                                         (
275                                                 SELECT is_anonymous_proxy FROM autnum_overrides overrides
276                                                         WHERE announcements.autnum = overrides.number
277                                         ),
278                                         FALSE
279                                 ) AS is_anonymous_proxy,
280                                 COALESCE(
281                                         (
282                                                 SELECT is_satellite_provider FROM network_overrides overrides
283                                                         WHERE announcements.network <<= overrides.network
284                                                         ORDER BY masklen(overrides.network) DESC
285                                                         LIMIT 1
286                                         ),
287                                         (
288                                                 SELECT is_satellite_provider FROM autnum_overrides overrides
289                                                         WHERE announcements.autnum = overrides.number
290                                         ),
291                                         FALSE
292                                 ) AS is_satellite_provider,
293                                 COALESCE(
294                                         (
295                                                 SELECT is_anycast FROM network_overrides overrides
296                                                         WHERE announcements.network <<= overrides.network
297                                                         ORDER BY masklen(overrides.network) DESC
298                                                         LIMIT 1
299                                         ),
300                                         (
301                                                 SELECT is_anycast FROM autnum_overrides overrides
302                                                         WHERE announcements.autnum = overrides.number
303                                         ),
304                                         FALSE
305                                 ) AS is_anycast,
306
307                                 -- Must be part of returned values for ORDER BY clause
308                                 masklen(known_networks.network) AS sort
309                         FROM known_networks
310                                 LEFT JOIN announcements ON known_networks.network <<= announcements.network
311                                 LEFT JOIN networks ON known_networks.network <<= networks.network
312                         ORDER BY known_networks.network, sort DESC
313                 """)
314
315                 for row in rows:
316                         network = writer.add_network(row.network)
317
318                         # Save country
319                         if row.country:
320                                 network.country_code = row.country
321
322                         # Save ASN
323                         if row.autnum:
324                                 network.asn = row.autnum
325
326                         # Set flags
327                         if row.is_anonymous_proxy:
328                                 network.set_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY)
329
330                         if row.is_satellite_provider:
331                                 network.set_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER)
332
333                         if row.is_anycast:
334                                 network.set_flag(location.NETWORK_FLAG_ANYCAST)
335
336                 # Add all countries
337                 log.info("Writing countries...")
338                 rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
339
340                 for row in rows:
341                         c = writer.add_country(row.country_code)
342                         c.continent_code = row.continent_code
343                         c.name = row.name
344
345                 # Write everything to file
346                 log.info("Writing database to file...")
347                 for file in ns.file:
348                         writer.write(file)
349
350         def handle_update_whois(self, ns):
351                 downloader = location.importer.Downloader()
352
353                 # Download all sources
354                 with self.db.transaction():
355                         # Create some temporary tables to store parsed data
356                         self.db.execute("""
357                                 CREATE TEMPORARY TABLE _autnums(number integer, organization text)
358                                         ON COMMIT DROP;
359                                 CREATE UNIQUE INDEX _autnums_number ON _autnums(number);
360
361                                 CREATE TEMPORARY TABLE _organizations(handle text, name text)
362                                         ON COMMIT DROP;
363                                 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
364                         """)
365
366                         for source in location.importer.WHOIS_SOURCES:
367                                 with downloader.request(source, return_blocks=True) as f:
368                                         for block in f:
369                                                 self._parse_block(block)
370
371                         self.db.execute("""
372                                 INSERT INTO autnums(number, name)
373                                         SELECT _autnums.number, _organizations.name FROM _autnums
374                                                 LEFT JOIN _organizations ON _autnums.organization = _organizations.handle
375                                 ON CONFLICT (number) DO UPDATE SET name = excluded.name;
376                         """)
377
378                 # Download all extended sources
379                 for source in location.importer.EXTENDED_SOURCES:
380                         with self.db.transaction():
381                                 # Download data
382                                 with downloader.request(source) as f:
383                                         for line in f:
384                                                 self._parse_line(line)
385
386         def _parse_block(self, block):
387                 # Get first line to find out what type of block this is
388                 line = block[0]
389
390                 # aut-num
391                 if line.startswith("aut-num:"):
392                         return self._parse_autnum_block(block)
393
394                 # organisation
395                 elif line.startswith("organisation:"):
396                         return self._parse_org_block(block)
397
398         def _parse_autnum_block(self, block):
399                 autnum = {}
400                 for line in block:
401                         # Split line
402                         key, val = split_line(line)
403
404                         if key == "aut-num":
405                                 m = re.match(r"^(AS|as)(\d+)", val)
406                                 if m:
407                                         autnum["asn"] = m.group(2)
408
409                         elif key == "org":
410                                 autnum[key] = val
411
412                 # Skip empty objects
413                 if not autnum:
414                         return
415
416                 # Insert into database
417                 self.db.execute("INSERT INTO _autnums(number, organization) \
418                         VALUES(%s, %s) ON CONFLICT (number) DO UPDATE SET \
419                                 organization = excluded.organization",
420                         autnum.get("asn"), autnum.get("org"),
421                 )
422
423         def _parse_org_block(self, block):
424                 org = {}
425                 for line in block:
426                         # Split line
427                         key, val = split_line(line)
428
429                         if key in ("organisation", "org-name"):
430                                 org[key] = val
431
432                 # Skip empty objects
433                 if not org:
434                         return
435
436                 self.db.execute("INSERT INTO _organizations(handle, name) \
437                         VALUES(%s, %s) ON CONFLICT (handle) DO \
438                         UPDATE SET name = excluded.name",
439                         org.get("organisation"), org.get("org-name"),
440                 )
441
442         def _parse_line(self, line):
443                 # Skip version line
444                 if line.startswith("2"):
445                         return
446
447                 # Skip comments
448                 if line.startswith("#"):
449                         return
450
451                 try:
452                         registry, country_code, type, line = line.split("|", 3)
453                 except:
454                         log.warning("Could not parse line: %s" % line)
455                         return
456
457                 # Skip any lines that are for stats only
458                 if country_code == "*":
459                         return
460
461                 if type in ("ipv6", "ipv4"):
462                         return self._parse_ip_line(country_code, type, line)
463
464         def _parse_ip_line(self, country, type, line):
465                 try:
466                         address, prefix, date, status, organization = line.split("|")
467                 except ValueError:
468                         organization = None
469
470                         # Try parsing the line without organization
471                         try:
472                                 address, prefix, date, status = line.split("|")
473                         except ValueError:
474                                 log.warning("Unhandled line format: %s" % line)
475                                 return
476
477                 # Skip anything that isn't properly assigned
478                 if not status in ("assigned", "allocated"):
479                         return
480
481                 # Cast prefix into an integer
482                 try:
483                         prefix = int(prefix)
484                 except:
485                         log.warning("Invalid prefix: %s" % prefix)
486                         return
487
488                 # Fix prefix length for IPv4
489                 if type == "ipv4":
490                         prefix = 32 - int(math.log(prefix, 2))
491
492                 # Try to parse the address
493                 try:
494                         network = ipaddress.ip_network("%s/%s" % (address, prefix), strict=False)
495                 except ValueError:
496                         log.warning("Invalid IP address: %s" % address)
497                         return
498
499                 self.db.execute("INSERT INTO networks(network, country) \
500                         VALUES(%s, %s) ON CONFLICT (network) DO \
501                         UPDATE SET country = excluded.country",
502                         "%s" % network, country,
503                 )
504
505         def handle_update_announcements(self, ns):
506                 server = ns.server[0]
507
508                 # Pre-compile regular expression for routes
509                 route = re.compile(b"^\*[\s\>]i([^\s]+).+?(\d+)\si\r\n", re.MULTILINE|re.DOTALL)
510
511                 with telnetlib.Telnet(server) as t:
512                         # Enable debug mode
513                         #if ns.debug:
514                         #       t.set_debuglevel(10)
515
516                         # Wait for console greeting
517                         greeting = t.read_until(b"> ", timeout=30)
518                         if not greeting:
519                                 log.error("Could not get a console prompt")
520                                 return 1
521
522                         # Disable pagination
523                         t.write(b"terminal length 0\n")
524
525                         # Wait for the prompt to return
526                         t.read_until(b"> ")
527
528                         # Fetch the routing tables
529                         with self.db.transaction():
530                                 for protocol in ("ipv6", "ipv4"):
531                                         log.info("Requesting %s routing table" % protocol)
532
533                                         # Request the full unicast routing table
534                                         t.write(b"show bgp %s unicast\n" % protocol.encode())
535
536                                         # Read entire header which ends with "Path"
537                                         t.read_until(b"Path\r\n")
538
539                                         while True:
540                                                 # Try reading a full entry
541                                                 # Those might be broken across multiple lines but ends with i
542                                                 line = t.read_until(b"i\r\n", timeout=5)
543                                                 if not line:
544                                                         break
545
546                                                 # Show line for debugging
547                                                 #log.debug(repr(line))
548
549                                                 # Try finding a route in here
550                                                 m = route.match(line)
551                                                 if m:
552                                                         network, autnum = m.groups()
553
554                                                         # Convert network to string
555                                                         network = network.decode()
556
557                                                         # Append /24 for IPv4 addresses
558                                                         if not "/" in network and not ":" in network:
559                                                                 network = "%s/24" % network
560
561                                                         # Convert AS number to integer
562                                                         autnum = int(autnum)
563
564                                                         log.info("Found announcement for %s by %s" % (network, autnum))
565
566                                                         self.db.execute("INSERT INTO announcements(network, autnum) \
567                                                                 VALUES(%s, %s) ON CONFLICT (network) DO \
568                                                                 UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
569                                                                 network, autnum,
570                                                         )
571
572                                         log.info("Finished reading the %s routing table" % protocol)
573
574                                 # Purge anything we never want here
575                                 self.db.execute("""
576                                         -- Delete default routes
577                                         DELETE FROM announcements WHERE network = '::/0' OR network = '0.0.0.0/0';
578
579                                         -- Delete anything that is not global unicast address space
580                                         DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3';
581
582                                         -- DELETE "current network" address space
583                                         DELETE FROM announcements WHERE family(network) = 4 AND network <<= '0.0.0.0/8';
584
585                                         -- DELETE local loopback address space
586                                         DELETE FROM announcements WHERE family(network) = 4 AND network <<= '127.0.0.0/8';
587
588                                         -- DELETE RFC 1918 address space
589                                         DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8';
590                                         DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12';
591                                         DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16';
592
593                                         -- DELETE test, benchmark and documentation address space
594                                         DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.0.0/24';
595                                         DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.2.0/24';
596                                         DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.18.0.0/15';
597                                         DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.51.100.0/24';
598                                         DELETE FROM announcements WHERE family(network) = 4 AND network <<= '203.0.113.0/24';
599
600                                         -- DELETE CGNAT address space (RFC 6598)
601                                         DELETE FROM announcements WHERE family(network) = 4 AND network <<= '100.64.0.0/10';
602
603                                         -- DELETE link local address space
604                                         DELETE FROM announcements WHERE family(network) = 4 AND network <<= '169.254.0.0/16';
605
606                                         -- DELETE IPv6 to IPv4 (6to4) address space
607                                         DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.88.99.0/24';
608
609                                         -- DELETE multicast and reserved address space
610                                         DELETE FROM announcements WHERE family(network) = 4 AND network <<= '224.0.0.0/4';
611                                         DELETE FROM announcements WHERE family(network) = 4 AND network <<= '240.0.0.0/4';
612
613                                         -- Delete networks that are too small to be in the global routing table
614                                         DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48;
615                                         DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24;
616
617                                         -- Delete any non-public or reserved ASNs
618                                         DELETE FROM announcements WHERE NOT (
619                                                 (autnum >= 1 AND autnum <= 23455)
620                                                 OR
621                                                 (autnum >= 23457 AND autnum <= 64495)
622                                                 OR
623                                                 (autnum >= 131072 AND autnum <= 4199999999)
624                                         );
625
626                                         -- Delete everything that we have not seen for 14 days
627                                         DELETE FROM announcements WHERE last_seen_at <= CURRENT_TIMESTAMP - INTERVAL '14 days';
628                                 """)
629
630         def handle_update_overrides(self, ns):
631                 with self.db.transaction():
632                         # Drop all data that we have
633                         self.db.execute("""
634                                 TRUNCATE TABLE autnum_overrides;
635                                 TRUNCATE TABLE network_overrides;
636                         """)
637
638                         for file in ns.files:
639                                 log.info("Reading %s..." % file)
640
641                                 with open(file, "rb") as f:
642                                         for type, block in location.importer.read_blocks(f):
643                                                 if type == "net":
644                                                         network = block.get("net")
645                                                         # Try to parse and normalise the network
646                                                         try:
647                                                                 network = ipaddress.ip_network(network, strict=False)
648                                                         except ValueError as e:
649                                                                 log.warning("Invalid IP network: %s: %s" % (network, e))
650                                                                 continue
651
652                                                         # Prevent that we overwrite all networks
653                                                         if network.prefixlen == 0:
654                                                                 log.warning("Skipping %s: You cannot overwrite default" % network)
655                                                                 continue
656
657                                                         self.db.execute("""
658                                                                 INSERT INTO network_overrides(
659                                                                         network,
660                                                                         country,
661                                                                         is_anonymous_proxy,
662                                                                         is_satellite_provider,
663                                                                         is_anycast
664                                                                 ) VALUES (%s, %s, %s, %s, %s)
665                                                                 ON CONFLICT (network) DO NOTHING""",
666                                                                 "%s" % network,
667                                                                 block.get("country"),
668                                                                 self._parse_bool(block, "is-anonymous-proxy"),
669                                                                 self._parse_bool(block, "is-satellite-provider"),
670                                                                 self._parse_bool(block, "is-anycast"),
671                                                         )
672
673                                                 elif type == "aut-num":
674                                                         autnum = block.get("aut-num")
675
676                                                         # Check if AS number begins with "AS"
677                                                         if not autnum.startswith("AS"):
678                                                                 log.warning("Invalid AS number: %s" % autnum)
679                                                                 continue
680
681                                                         # Strip "AS"
682                                                         autnum = autnum[2:]
683
684                                                         self.db.execute("""
685                                                                 INSERT INTO autnum_overrides(
686                                                                         number,
687                                                                         name,
688                                                                         country,
689                                                                         is_anonymous_proxy,
690                                                                         is_satellite_provider,
691                                                                         is_anycast
692                                                                 ) VALUES(%s, %s, %s, %s, %s, %s)
693                                                                 ON CONFLICT DO NOTHING""",
694                                                                 autnum,
695                                                                 block.get("name"),
696                                                                 block.get("country"),
697                                                                 self._parse_bool(block, "is-anonymous-proxy"),
698                                                                 self._parse_bool(block, "is-satellite-provider"),
699                                                                 self._parse_bool(block, "is-anycast"),
700                                                         )
701
702                                                 else:
703                                                         log.warning("Unsupport type: %s" % type)
704
705         @staticmethod
706         def _parse_bool(block, key):
707                 val = block.get(key)
708
709                 # There is no point to proceed when we got None
710                 if val is None:
711                         return
712
713                 # Convert to lowercase
714                 val = val.lower()
715
716                 # True
717                 if val in ("yes", "1"):
718                         return True
719
720                 # False
721                 if val in ("no", "0"):
722                         return False
723
724                 # Default to None
725                 return None
726
727         def handle_import_countries(self, ns):
728                 with self.db.transaction():
729                         # Drop all data that we have
730                         self.db.execute("TRUNCATE TABLE countries")
731
732                         for file in ns.file:
733                                 for line in file:
734                                         line = line.rstrip()
735
736                                         # Ignore any comments
737                                         if line.startswith("#"):
738                                                 continue
739
740                                         try:
741                                                 country_code, continent_code, name = line.split(maxsplit=2)
742                                         except:
743                                                 log.warning("Could not parse line: %s" % line)
744                                                 continue
745
746                                         self.db.execute("INSERT INTO countries(country_code, name, continent_code) \
747                                                 VALUES(%s, %s, %s) ON CONFLICT DO NOTHING", country_code, name, continent_code)
748
749
750 def split_line(line):
751         key, colon, val = line.partition(":")
752
753         # Strip any excess space
754         key = key.strip()
755         val = val.strip()
756
757         return key, val
758
759 def main():
760         # Run the command line interface
761         c = CLI()
762         c.run()
763
764 main()