]> git.ipfire.org Git - location/libloc.git/blob - src/python/location-importer.in
projects / location / libloc.git / blob
? search:


d0fe5a69592ae57cf38fd1118a1fd774e6130ed5
[location/libloc.git] / src / python / location-importer.in
1 #!/usr/bin/python3
2 ###############################################################################
3 # #
4 # libloc - A library to determine the location of someone on the Internet #
5 # #
6 # Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
7 # #
8 # This library is free software; you can redistribute it and/or #
9 # modify it under the terms of the GNU Lesser General Public #
10 # License as published by the Free Software Foundation; either #
11 # version 2.1 of the License, or (at your option) any later version. #
12 # #
13 # This library is distributed in the hope that it will be useful, #
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
16 # Lesser General Public License for more details. #
17 # #
18 ###############################################################################
19
20 import argparse
21 import ipaddress
22 import logging
23 import math
24 import re
25 import socket
26 import sys
27 import telnetlib
28
29 # Load our location module
30 import location
31 import location.database
32 import location.importer
33 from location.i18n import _
34
35 # Initialise logging
36 log = logging.getLogger("location.importer")
37 log.propagate = 1
38
39 class CLI(object):
40 def parse_cli(self):
41 parser = argparse.ArgumentParser(
42 description=_("Location Importer Command Line Interface"),
43 )
44 subparsers = parser.add_subparsers()
45
46 # Global configuration flags
47 parser.add_argument("--debug", action="store_true",
48 help=_("Enable debug output"))
49 parser.add_argument("--quiet", action="store_true",
50 help=_("Enable quiet mode"))
51
52 # version
53 parser.add_argument("--version", action="version",
54 version="%(prog)s @VERSION@")
55
56 # Database
57 parser.add_argument("--database-host", required=True,
58 help=_("Database Hostname"), metavar=_("HOST"))
59 parser.add_argument("--database-name", required=True,
60 help=_("Database Name"), metavar=_("NAME"))
61 parser.add_argument("--database-username", required=True,
62 help=_("Database Username"), metavar=_("USERNAME"))
63 parser.add_argument("--database-password", required=True,
64 help=_("Database Password"), metavar=_("PASSWORD"))
65
66 # Write Database
67 write = subparsers.add_parser("write", help=_("Write database to file"))
68 write.set_defaults(func=self.handle_write)
69 write.add_argument("file", nargs=1, help=_("Database File"))
70 write.add_argument("--signing-key", nargs="?", type=open, help=_("Signing Key"))
71 write.add_argument("--backup-signing-key", nargs="?", type=open, help=_("Backup Signing Key"))
72 write.add_argument("--vendor", nargs="?", help=_("Sets the vendor"))
73 write.add_argument("--description", nargs="?", help=_("Sets a description"))
74 write.add_argument("--license", nargs="?", help=_("Sets the license"))
75 write.add_argument("--version", type=int, help=_("Database Format Version"))
76
77 # Update WHOIS
78 update_whois = subparsers.add_parser("update-whois", help=_("Update WHOIS Information"))
79 update_whois.set_defaults(func=self.handle_update_whois)
80
81 # Update announcements
82 update_announcements = subparsers.add_parser("update-announcements",
83 help=_("Update BGP Annoucements"))
84 update_announcements.set_defaults(func=self.handle_update_announcements)
85 update_announcements.add_argument("server", nargs=1,
86 help=_("Route Server to connect to"), metavar=_("SERVER"))
87
88 # Update overrides
89 update_overrides = subparsers.add_parser("update-overrides",
90 help=_("Update overrides"),
91 )
92 update_overrides.add_argument(
93 "files", nargs="+", help=_("Files to import"),
94 )
95 update_overrides.set_defaults(func=self.handle_update_overrides)
96
97 # Import countries
98 import_countries = subparsers.add_parser("import-countries",
99 help=_("Import countries"),
100 )
101 import_countries.add_argument("file", nargs=1, type=argparse.FileType("r"),
102 help=_("File to import"))
103 import_countries.set_defaults(func=self.handle_import_countries)
104
105 args = parser.parse_args()
106
107 # Configure logging
108 if args.debug:
109 location.logger.set_level(logging.DEBUG)
110 elif args.quiet:
111 location.logger.set_level(logging.WARNING)
112
113 # Print usage if no action was given
114 if not "func" in args:
115 parser.print_usage()
116 sys.exit(2)
117
118 return args
119
120 def run(self):
121 # Parse command line arguments
122 args = self.parse_cli()
123
124 # Initialise database
125 self.db = self._setup_database(args)
126
127 # Call function
128 ret = args.func(args)
129
130 # Return with exit code
131 if ret:
132 sys.exit(ret)
133
134 # Otherwise just exit
135 sys.exit(0)
136
137 def _setup_database(self, ns):
138 """
139 Initialise the database
140 """
141 # Connect to database
142 db = location.database.Connection(
143 host=ns.database_host, database=ns.database_name,
144 user=ns.database_username, password=ns.database_password,
145 )
146
147 with db.transaction():
148 db.execute("""
149 -- announcements
150 CREATE TABLE IF NOT EXISTS announcements(network inet, autnum bigint,
151 first_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP,
152 last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
153 CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
154 CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
155 CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops);
156
157 -- autnums
158 CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
159 CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number);
160
161 -- countries
162 CREATE TABLE IF NOT EXISTS countries(
163 country_code text NOT NULL, name text NOT NULL, continent_code text NOT NULL);
164 CREATE UNIQUE INDEX IF NOT EXISTS countries_country_code ON countries(country_code);
165
166 -- networks
167 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
168 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
169 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
170
171 -- overrides
172 CREATE TABLE IF NOT EXISTS autnum_overrides(
173 number bigint NOT NULL,
174 name text,
175 country text,
176 is_anonymous_proxy boolean,
177 is_satellite_provider boolean,
178 is_anycast boolean
179 );
180 CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number
181 ON autnum_overrides(number);
182
183 CREATE TABLE IF NOT EXISTS network_overrides(
184 network inet NOT NULL,
185 country text,
186 is_anonymous_proxy boolean,
187 is_satellite_provider boolean,
188 is_anycast boolean
189 );
190 CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
191 ON network_overrides(network);
192 """)
193
194 return db
195
196 def handle_write(self, ns):
197 """
198 Compiles a database in libloc format out of what is in the database
199 """
200 # Allocate a writer
201 writer = location.Writer(ns.signing_key, ns.backup_signing_key)
202
203 # Set all metadata
204 if ns.vendor:
205 writer.vendor = ns.vendor
206
207 if ns.description:
208 writer.description = ns.description
209
210 if ns.license:
211 writer.license = ns.license
212
213 # Add all Autonomous Systems
214 log.info("Writing Autonomous Systems...")
215
216 # Select all ASes with a name
217 rows = self.db.query("""
218 SELECT
219 autnums.number AS number,
220 COALESCE(
221 (SELECT overrides.name FROM autnum_overrides overrides
222 WHERE overrides.number = autnums.number),
223 autnums.name
224 ) AS name
225 FROM autnums
226 WHERE name <> %s ORDER BY number
227 """, "")
228
229 for row in rows:
230 a = writer.add_as(row.number)
231 a.name = row.name
232
233 # Add all networks
234 log.info("Writing networks...")
235
236 # Select all known networks
237 rows = self.db.query("""
238 -- Get a (sorted) list of all known networks
239 WITH known_networks AS (
240 SELECT network FROM announcements
241 UNION
242 SELECT network FROM networks
243 UNION
244 SELECT network FROM network_overrides
245 ORDER BY network
246 )
247
248 -- Return a list of those networks enriched with all
249 -- other information that we store in the database
250 SELECT
251 DISTINCT ON (known_networks.network)
252 known_networks.network AS network,
253 announcements.autnum AS autnum,
254
255 -- Country
256 COALESCE(
257 (
258 SELECT country FROM network_overrides overrides
259 WHERE announcements.network <<= overrides.network
260 ORDER BY masklen(overrides.network) DESC
261 LIMIT 1
262 ),
263 (
264 SELECT country FROM autnum_overrides overrides
265 WHERE announcements.autnum = overrides.number
266 ),
267 networks.country
268 ) AS country,
269
270 -- Flags
271 COALESCE(
272 (
273 SELECT is_anonymous_proxy FROM network_overrides overrides
274 WHERE announcements.network <<= overrides.network
275 ORDER BY masklen(overrides.network) DESC
276 LIMIT 1
277 ),
278 (
279 SELECT is_anonymous_proxy FROM autnum_overrides overrides
280 WHERE announcements.autnum = overrides.number
281 ),
282 FALSE
283 ) AS is_anonymous_proxy,
284 COALESCE(
285 (
286 SELECT is_satellite_provider FROM network_overrides overrides
287 WHERE announcements.network <<= overrides.network
288 ORDER BY masklen(overrides.network) DESC
289 LIMIT 1
290 ),
291 (
292 SELECT is_satellite_provider FROM autnum_overrides overrides
293 WHERE announcements.autnum = overrides.number
294 ),
295 FALSE
296 ) AS is_satellite_provider,
297 COALESCE(
298 (
299 SELECT is_anycast FROM network_overrides overrides
300 WHERE announcements.network <<= overrides.network
301 ORDER BY masklen(overrides.network) DESC
302 LIMIT 1
303 ),
304 (
305 SELECT is_anycast FROM autnum_overrides overrides
306 WHERE announcements.autnum = overrides.number
307 ),
308 FALSE
309 ) AS is_anycast,
310
311 -- Must be part of returned values for ORDER BY clause
312 masklen(announcements.network) AS sort_a,
313 masklen(networks.network) AS sort_b
314 FROM known_networks
315 LEFT JOIN announcements ON known_networks.network <<= announcements.network
316 LEFT JOIN networks ON known_networks.network <<= networks.network
317 ORDER BY known_networks.network, sort_a DESC, sort_b DESC
318 """)
319
320 for row in rows:
321 network = writer.add_network(row.network)
322
323 # Save country
324 if row.country:
325 network.country_code = row.country
326
327 # Save ASN
328 if row.autnum:
329 network.asn = row.autnum
330
331 # Set flags
332 if row.is_anonymous_proxy:
333 network.set_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY)
334
335 if row.is_satellite_provider:
336 network.set_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER)
337
338 if row.is_anycast:
339 network.set_flag(location.NETWORK_FLAG_ANYCAST)
340
341 # Add all countries
342 log.info("Writing countries...")
343 rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
344
345 for row in rows:
346 c = writer.add_country(row.country_code)
347 c.continent_code = row.continent_code
348 c.name = row.name
349
350 # Write everything to file
351 log.info("Writing database to file...")
352 for file in ns.file:
353 writer.write(file)
354
355 def handle_update_whois(self, ns):
356 downloader = location.importer.Downloader()
357
358 # Download all sources
359 with self.db.transaction():
360 # Create some temporary tables to store parsed data
361 self.db.execute("""
362 CREATE TEMPORARY TABLE _autnums(number integer, organization text)
363 ON COMMIT DROP;
364 CREATE UNIQUE INDEX _autnums_number ON _autnums(number);
365
366 CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
367 ON COMMIT DROP;
368 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
369 """)
370
371 for source in location.importer.WHOIS_SOURCES:
372 with downloader.request(source, return_blocks=True) as f:
373 for block in f:
374 self._parse_block(block)
375
376 self.db.execute("""
377 INSERT INTO autnums(number, name)
378 SELECT _autnums.number, _organizations.name FROM _autnums
379 JOIN _organizations ON _autnums.organization = _organizations.handle
380 ON CONFLICT (number) DO UPDATE SET name = excluded.name;
381 """)
382
383 # Download all extended sources
384 for source in location.importer.EXTENDED_SOURCES:
385 with self.db.transaction():
386 # Download data
387 with downloader.request(source) as f:
388 for line in f:
389 self._parse_line(line)
390
391 def _parse_block(self, block):
392 # Get first line to find out what type of block this is
393 line = block[0]
394
395 # aut-num
396 if line.startswith("aut-num:"):
397 return self._parse_autnum_block(block)
398
399 # organisation
400 elif line.startswith("organisation:"):
401 return self._parse_org_block(block)
402
403 def _parse_autnum_block(self, block):
404 autnum = {}
405 for line in block:
406 # Split line
407 key, val = split_line(line)
408
409 if key == "aut-num":
410 m = re.match(r"^(AS|as)(\d+)", val)
411 if m:
412 autnum["asn"] = m.group(2)
413
414 elif key == "org":
415 autnum[key] = val
416
417 # Skip empty objects
418 if not autnum:
419 return
420
421 # Insert into database
422 self.db.execute("INSERT INTO _autnums(number, organization) \
423 VALUES(%s, %s) ON CONFLICT (number) DO UPDATE SET \
424 organization = excluded.organization",
425 autnum.get("asn"), autnum.get("org"),
426 )
427
428 def _parse_org_block(self, block):
429 org = {}
430 for line in block:
431 # Split line
432 key, val = split_line(line)
433
434 if key in ("organisation", "org-name"):
435 org[key] = val
436
437 # Skip empty objects
438 if not org:
439 return
440
441 self.db.execute("INSERT INTO _organizations(handle, name) \
442 VALUES(%s, %s) ON CONFLICT (handle) DO \
443 UPDATE SET name = excluded.name",
444 org.get("organisation"), org.get("org-name"),
445 )
446
447 def _parse_line(self, line):
448 # Skip version line
449 if line.startswith("2"):
450 return
451
452 # Skip comments
453 if line.startswith("#"):
454 return
455
456 try:
457 registry, country_code, type, line = line.split("|", 3)
458 except:
459 log.warning("Could not parse line: %s" % line)
460 return
461
462 # Skip any lines that are for stats only
463 if country_code == "*":
464 return
465
466 if type in ("ipv6", "ipv4"):
467 return self._parse_ip_line(country_code, type, line)
468
469 def _parse_ip_line(self, country, type, line):
470 try:
471 address, prefix, date, status, organization = line.split("|")
472 except ValueError:
473 organization = None
474
475 # Try parsing the line without organization
476 try:
477 address, prefix, date, status = line.split("|")
478 except ValueError:
479 log.warning("Unhandled line format: %s" % line)
480 return
481
482 # Skip anything that isn't properly assigned
483 if not status in ("assigned", "allocated"):
484 return
485
486 # Cast prefix into an integer
487 try:
488 prefix = int(prefix)
489 except:
490 log.warning("Invalid prefix: %s" % prefix)
491 return
492
493 # Fix prefix length for IPv4
494 if type == "ipv4":
495 prefix = 32 - int(math.log(prefix, 2))
496
497 # Try to parse the address
498 try:
499 network = ipaddress.ip_network("%s/%s" % (address, prefix), strict=False)
500 except ValueError:
501 log.warning("Invalid IP address: %s" % address)
502 return
503
504 self.db.execute("INSERT INTO networks(network, country) \
505 VALUES(%s, %s) ON CONFLICT (network) DO \
506 UPDATE SET country = excluded.country",
507 "%s" % network, country,
508 )
509
510 def handle_update_announcements(self, ns):
511 server = ns.server[0]
512
513 with self.db.transaction():
514 if server.startswith("/"):
515 self._handle_update_announcements_from_bird(server)
516 else:
517 self._handle_update_announcements_from_telnet(server)
518
519 # Purge anything we never want here
520 self.db.execute("""
521 -- Delete default routes
522 DELETE FROM announcements WHERE network = '::/0' OR network = '0.0.0.0/0';
523
524 -- Delete anything that is not global unicast address space
525 DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3';
526
527 -- DELETE "current network" address space
528 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '0.0.0.0/8';
529
530 -- DELETE local loopback address space
531 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '127.0.0.0/8';
532
533 -- DELETE RFC 1918 address space
534 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8';
535 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12';
536 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16';
537
538 -- DELETE test, benchmark and documentation address space
539 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.0.0/24';
540 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.2.0/24';
541 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.18.0.0/15';
542 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.51.100.0/24';
543 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '203.0.113.0/24';
544
545 -- DELETE CGNAT address space (RFC 6598)
546 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '100.64.0.0/10';
547
548 -- DELETE link local address space
549 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '169.254.0.0/16';
550
551 -- DELETE IPv6 to IPv4 (6to4) address space
552 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.88.99.0/24';
553
554 -- DELETE multicast and reserved address space
555 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '224.0.0.0/4';
556 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '240.0.0.0/4';
557
558 -- Delete networks that are too small to be in the global routing table
559 DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48;
560 DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24;
561
562 -- Delete any non-public or reserved ASNs
563 DELETE FROM announcements WHERE NOT (
564 (autnum >= 1 AND autnum <= 23455)
565 OR
566 (autnum >= 23457 AND autnum <= 64495)
567 OR
568 (autnum >= 131072 AND autnum <= 4199999999)
569 );
570
571 -- Delete everything that we have not seen for 14 days
572 DELETE FROM announcements WHERE last_seen_at <= CURRENT_TIMESTAMP - INTERVAL '14 days';
573 """)
574
575 def _handle_update_announcements_from_bird(self, server):
576 # Pre-compile the regular expression for faster searching
577 route = re.compile(b"^\s(.+?)\s+.+?\[AS(.*?).\]$")
578
579 log.info("Requesting routing table from Bird (%s)" % server)
580
581 # Send command to list all routes
582 for line in self._bird_cmd(server, "show route"):
583 m = route.match(line)
584 if not m:
585 log.debug("Could not parse line: %s" % line.decode())
586 continue
587
588 # Fetch the extracted network and ASN
589 network, autnum = m.groups()
590
591 # Insert it into the database
592 self.db.execute("INSERT INTO announcements(network, autnum) \
593 VALUES(%s, %s) ON CONFLICT (network) DO \
594 UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
595 network.decode(), autnum.decode(),
596 )
597
598 def _handle_update_announcements_from_telnet(self, server):
599 # Pre-compile regular expression for routes
600 route = re.compile(b"^\*[\s\>]i([^\s]+).+?(\d+)\si\r\n", re.MULTILINE|re.DOTALL)
601
602 with telnetlib.Telnet(server) as t:
603 # Enable debug mode
604 #if ns.debug:
605 # t.set_debuglevel(10)
606
607 # Wait for console greeting
608 greeting = t.read_until(b"> ", timeout=30)
609 if not greeting:
610 log.error("Could not get a console prompt")
611 return 1
612
613 # Disable pagination
614 t.write(b"terminal length 0\n")
615
616 # Wait for the prompt to return
617 t.read_until(b"> ")
618
619 # Fetch the routing tables
620 for protocol in ("ipv6", "ipv4"):
621 log.info("Requesting %s routing table" % protocol)
622
623 # Request the full unicast routing table
624 t.write(b"show bgp %s unicast\n" % protocol.encode())
625
626 # Read entire header which ends with "Path"
627 t.read_until(b"Path\r\n")
628
629 while True:
630 # Try reading a full entry
631 # Those might be broken across multiple lines but ends with i
632 line = t.read_until(b"i\r\n", timeout=5)
633 if not line:
634 break
635
636 # Show line for debugging
637 #log.debug(repr(line))
638
639 # Try finding a route in here
640 m = route.match(line)
641 if m:
642 network, autnum = m.groups()
643
644 # Convert network to string
645 network = network.decode()
646
647 # Append /24 for IPv4 addresses
648 if not "/" in network and not ":" in network:
649 network = "%s/24" % network
650
651 # Convert AS number to integer
652 autnum = int(autnum)
653
654 log.info("Found announcement for %s by %s" % (network, autnum))
655
656 self.db.execute("INSERT INTO announcements(network, autnum) \
657 VALUES(%s, %s) ON CONFLICT (network) DO \
658 UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
659 network, autnum,
660 )
661
662 log.info("Finished reading the %s routing table" % protocol)
663
664 def _bird_cmd(self, socket_path, command):
665 # Connect to the socket
666 s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
667 s.connect(socket_path)
668
669 # Allocate some buffer
670 buffer = b""
671
672 # Send the command
673 s.send(b"%s\n" % command.encode())
674
675 while True:
676 # Fill up the buffer
677 buffer += s.recv(4096)
678
679 while True:
680 # Search for the next newline
681 pos = buffer.find(b"\n")
682
683 # If we cannot find one, we go back and read more data
684 if pos <= 0:
685 break
686
687 # Cut after the newline character
688 pos += 1
689
690 # Split the line we want and keep the rest in buffer
691 line, buffer = buffer[:pos], buffer[pos:]
692
693 # Look for the end-of-output indicator
694 if line == b"0000 \n":
695 return
696
697 # Otherwise return the line
698 yield line
699
700 def handle_update_overrides(self, ns):
701 with self.db.transaction():
702 # Drop all data that we have
703 self.db.execute("""
704 TRUNCATE TABLE autnum_overrides;
705 TRUNCATE TABLE network_overrides;
706 """)
707
708 for file in ns.files:
709 log.info("Reading %s..." % file)
710
711 with open(file, "rb") as f:
712 for type, block in location.importer.read_blocks(f):
713 if type == "net":
714 network = block.get("net")
715 # Try to parse and normalise the network
716 try:
717 network = ipaddress.ip_network(network, strict=False)
718 except ValueError as e:
719 log.warning("Invalid IP network: %s: %s" % (network, e))
720 continue
721
722 # Prevent that we overwrite all networks
723 if network.prefixlen == 0:
724 log.warning("Skipping %s: You cannot overwrite default" % network)
725 continue
726
727 self.db.execute("""
728 INSERT INTO network_overrides(
729 network,
730 country,
731 is_anonymous_proxy,
732 is_satellite_provider,
733 is_anycast
734 ) VALUES (%s, %s, %s, %s, %s)
735 ON CONFLICT (network) DO NOTHING""",
736 "%s" % network,
737 block.get("country"),
738 self._parse_bool(block, "is-anonymous-proxy"),
739 self._parse_bool(block, "is-satellite-provider"),
740 self._parse_bool(block, "is-anycast"),
741 )
742
743 elif type == "aut-num":
744 autnum = block.get("aut-num")
745
746 # Check if AS number begins with "AS"
747 if not autnum.startswith("AS"):
748 log.warning("Invalid AS number: %s" % autnum)
749 continue
750
751 # Strip "AS"
752 autnum = autnum[2:]
753
754 self.db.execute("""
755 INSERT INTO autnum_overrides(
756 number,
757 name,
758 country,
759 is_anonymous_proxy,
760 is_satellite_provider,
761 is_anycast
762 ) VALUES(%s, %s, %s, %s, %s, %s)
763 ON CONFLICT DO NOTHING""",
764 autnum,
765 block.get("name"),
766 block.get("country"),
767 self._parse_bool(block, "is-anonymous-proxy"),
768 self._parse_bool(block, "is-satellite-provider"),
769 self._parse_bool(block, "is-anycast"),
770 )
771
772 else:
773 log.warning("Unsupport type: %s" % type)
774
775 @staticmethod
776 def _parse_bool(block, key):
777 val = block.get(key)
778
779 # There is no point to proceed when we got None
780 if val is None:
781 return
782
783 # Convert to lowercase
784 val = val.lower()
785
786 # True
787 if val in ("yes", "1"):
788 return True
789
790 # False
791 if val in ("no", "0"):
792 return False
793
794 # Default to None
795 return None
796
797 def handle_import_countries(self, ns):
798 with self.db.transaction():
799 # Drop all data that we have
800 self.db.execute("TRUNCATE TABLE countries")
801
802 for file in ns.file:
803 for line in file:
804 line = line.rstrip()
805
806 # Ignore any comments
807 if line.startswith("#"):
808 continue
809
810 try:
811 country_code, continent_code, name = line.split(maxsplit=2)
812 except:
813 log.warning("Could not parse line: %s" % line)
814 continue
815
816 self.db.execute("INSERT INTO countries(country_code, name, continent_code) \
817 VALUES(%s, %s, %s) ON CONFLICT DO NOTHING", country_code, name, continent_code)
818
819
820 def split_line(line):
821 key, colon, val = line.partition(":")
822
823 # Strip any excess space
824 key = key.strip()
825 val = val.strip()
826
827 return key, val
828
829 def main():
830 # Run the command line interface
831 c = CLI()
832 c.run()
833
834 main()
A library to determine the location of someone on the Internet