]> git.ipfire.org Git - people/ms/libloc.git/blob - src/python/location-importer.in
projects / people / ms / libloc.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
importer: Purge any redundant entries
[people/ms/libloc.git] / src / python / location-importer.in
1 #!/usr/bin/python3
2 ###############################################################################
3 # #
4 # libloc - A library to determine the location of someone on the Internet #
5 # #
6 # Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
7 # #
8 # This library is free software; you can redistribute it and/or #
9 # modify it under the terms of the GNU Lesser General Public #
10 # License as published by the Free Software Foundation; either #
11 # version 2.1 of the License, or (at your option) any later version. #
12 # #
13 # This library is distributed in the hope that it will be useful, #
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
16 # Lesser General Public License for more details. #
17 # #
18 ###############################################################################
19
20 import argparse
21 import ipaddress
22 import logging
23 import math
24 import re
25 import socket
26 import sys
27 import telnetlib
28
29 # Load our location module
30 import location
31 import location.database
32 import location.importer
33 from location.i18n import _
34
35 # Initialise logging
36 log = logging.getLogger("location.importer")
37 log.propagate = 1
38
39 class CLI(object):
40 def parse_cli(self):
41 parser = argparse.ArgumentParser(
42 description=_("Location Importer Command Line Interface"),
43 )
44 subparsers = parser.add_subparsers()
45
46 # Global configuration flags
47 parser.add_argument("--debug", action="store_true",
48 help=_("Enable debug output"))
49 parser.add_argument("--quiet", action="store_true",
50 help=_("Enable quiet mode"))
51
52 # version
53 parser.add_argument("--version", action="version",
54 version="%(prog)s @VERSION@")
55
56 # Database
57 parser.add_argument("--database-host", required=True,
58 help=_("Database Hostname"), metavar=_("HOST"))
59 parser.add_argument("--database-name", required=True,
60 help=_("Database Name"), metavar=_("NAME"))
61 parser.add_argument("--database-username", required=True,
62 help=_("Database Username"), metavar=_("USERNAME"))
63 parser.add_argument("--database-password", required=True,
64 help=_("Database Password"), metavar=_("PASSWORD"))
65
66 # Write Database
67 write = subparsers.add_parser("write", help=_("Write database to file"))
68 write.set_defaults(func=self.handle_write)
69 write.add_argument("file", nargs=1, help=_("Database File"))
70 write.add_argument("--signing-key", nargs="?", type=open, help=_("Signing Key"))
71 write.add_argument("--backup-signing-key", nargs="?", type=open, help=_("Backup Signing Key"))
72 write.add_argument("--vendor", nargs="?", help=_("Sets the vendor"))
73 write.add_argument("--description", nargs="?", help=_("Sets a description"))
74 write.add_argument("--license", nargs="?", help=_("Sets the license"))
75 write.add_argument("--version", type=int, help=_("Database Format Version"))
76
77 # Update WHOIS
78 update_whois = subparsers.add_parser("update-whois", help=_("Update WHOIS Information"))
79 update_whois.set_defaults(func=self.handle_update_whois)
80
81 # Update announcements
82 update_announcements = subparsers.add_parser("update-announcements",
83 help=_("Update BGP Annoucements"))
84 update_announcements.set_defaults(func=self.handle_update_announcements)
85 update_announcements.add_argument("server", nargs=1,
86 help=_("Route Server to connect to"), metavar=_("SERVER"))
87
88 # Update overrides
89 update_overrides = subparsers.add_parser("update-overrides",
90 help=_("Update overrides"),
91 )
92 update_overrides.add_argument(
93 "files", nargs="+", help=_("Files to import"),
94 )
95 update_overrides.set_defaults(func=self.handle_update_overrides)
96
97 # Import countries
98 import_countries = subparsers.add_parser("import-countries",
99 help=_("Import countries"),
100 )
101 import_countries.add_argument("file", nargs=1, type=argparse.FileType("r"),
102 help=_("File to import"))
103 import_countries.set_defaults(func=self.handle_import_countries)
104
105 args = parser.parse_args()
106
107 # Configure logging
108 if args.debug:
109 location.logger.set_level(logging.DEBUG)
110 elif args.quiet:
111 location.logger.set_level(logging.WARNING)
112
113 # Print usage if no action was given
114 if not "func" in args:
115 parser.print_usage()
116 sys.exit(2)
117
118 return args
119
120 def run(self):
121 # Parse command line arguments
122 args = self.parse_cli()
123
124 # Initialise database
125 self.db = self._setup_database(args)
126
127 # Call function
128 ret = args.func(args)
129
130 # Return with exit code
131 if ret:
132 sys.exit(ret)
133
134 # Otherwise just exit
135 sys.exit(0)
136
137 def _setup_database(self, ns):
138 """
139 Initialise the database
140 """
141 # Connect to database
142 db = location.database.Connection(
143 host=ns.database_host, database=ns.database_name,
144 user=ns.database_username, password=ns.database_password,
145 )
146
147 with db.transaction():
148 db.execute("""
149 -- announcements
150 CREATE TABLE IF NOT EXISTS announcements(network inet, autnum bigint,
151 first_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP,
152 last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
153 CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
154 CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
155
156 -- autnums
157 CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
158 CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number);
159
160 -- countries
161 CREATE TABLE IF NOT EXISTS countries(
162 country_code text NOT NULL, name text NOT NULL, continent_code text NOT NULL);
163 CREATE UNIQUE INDEX IF NOT EXISTS countries_country_code ON countries(country_code);
164
165 -- networks
166 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
167 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
168 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
169
170 -- overrides
171 CREATE TABLE IF NOT EXISTS autnum_overrides(
172 number bigint NOT NULL,
173 name text,
174 country text,
175 is_anonymous_proxy boolean,
176 is_satellite_provider boolean,
177 is_anycast boolean
178 );
179 CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number
180 ON autnum_overrides(number);
181
182 CREATE TABLE IF NOT EXISTS network_overrides(
183 network inet NOT NULL,
184 country text,
185 is_anonymous_proxy boolean,
186 is_satellite_provider boolean,
187 is_anycast boolean
188 );
189 CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
190 ON network_overrides(network);
191 """)
192
193 return db
194
195 def handle_write(self, ns):
196 """
197 Compiles a database in libloc format out of what is in the database
198 """
199 # Allocate a writer
200 writer = location.Writer(ns.signing_key, ns.backup_signing_key)
201
202 # Set all metadata
203 if ns.vendor:
204 writer.vendor = ns.vendor
205
206 if ns.description:
207 writer.description = ns.description
208
209 if ns.license:
210 writer.license = ns.license
211
212 # Add all Autonomous Systems
213 log.info("Writing Autonomous Systems...")
214
215 # Select all ASes with a name
216 rows = self.db.query("""
217 SELECT
218 autnums.number AS number,
219 COALESCE(
220 (SELECT overrides.name FROM autnum_overrides overrides
221 WHERE overrides.number = autnums.number),
222 autnums.name
223 ) AS name
224 FROM autnums
225 WHERE name <> %s ORDER BY number
226 """, "")
227
228 for row in rows:
229 a = writer.add_as(row.number)
230 a.name = row.name
231
232 # Add all networks
233 log.info("Writing networks...")
234
235 # Select all known networks
236 rows = self.db.query("""
237 -- Get a (sorted) list of all known networks
238 WITH known_networks AS (
239 SELECT network FROM announcements
240 UNION
241 SELECT network FROM networks
242 ORDER BY network
243 )
244
245 -- Return a list of those networks enriched with all
246 -- other information that we store in the database
247 SELECT
248 DISTINCT ON (known_networks.network)
249 known_networks.network AS network,
250 announcements.autnum AS autnum,
251
252 -- Country
253 COALESCE(
254 (
255 SELECT country FROM network_overrides overrides
256 WHERE announcements.network <<= overrides.network
257 ORDER BY masklen(overrides.network) DESC
258 LIMIT 1
259 ),
260 (
261 SELECT country FROM autnum_overrides overrides
262 WHERE announcements.autnum = overrides.number
263 ),
264 networks.country
265 ) AS country,
266
267 -- Flags
268 COALESCE(
269 (
270 SELECT is_anonymous_proxy FROM network_overrides overrides
271 WHERE announcements.network <<= overrides.network
272 ORDER BY masklen(overrides.network) DESC
273 LIMIT 1
274 ),
275 (
276 SELECT is_anonymous_proxy FROM autnum_overrides overrides
277 WHERE announcements.autnum = overrides.number
278 ),
279 FALSE
280 ) AS is_anonymous_proxy,
281 COALESCE(
282 (
283 SELECT is_satellite_provider FROM network_overrides overrides
284 WHERE announcements.network <<= overrides.network
285 ORDER BY masklen(overrides.network) DESC
286 LIMIT 1
287 ),
288 (
289 SELECT is_satellite_provider FROM autnum_overrides overrides
290 WHERE announcements.autnum = overrides.number
291 ),
292 FALSE
293 ) AS is_satellite_provider,
294 COALESCE(
295 (
296 SELECT is_anycast FROM network_overrides overrides
297 WHERE announcements.network <<= overrides.network
298 ORDER BY masklen(overrides.network) DESC
299 LIMIT 1
300 ),
301 (
302 SELECT is_anycast FROM autnum_overrides overrides
303 WHERE announcements.autnum = overrides.number
304 ),
305 FALSE
306 ) AS is_anycast,
307
308 -- Must be part of returned values for ORDER BY clause
309 masklen(announcements.network) AS sort_a,
310 masklen(networks.network) AS sort_b
311 FROM known_networks
312 LEFT JOIN announcements ON known_networks.network <<= announcements.network
313 LEFT JOIN networks ON known_networks.network <<= networks.network
314 ORDER BY known_networks.network, sort_a DESC, sort_b DESC
315 """)
316
317 for row in rows:
318 network = writer.add_network(row.network)
319
320 # Save country
321 if row.country:
322 network.country_code = row.country
323
324 # Save ASN
325 if row.autnum:
326 network.asn = row.autnum
327
328 # Set flags
329 if row.is_anonymous_proxy:
330 network.set_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY)
331
332 if row.is_satellite_provider:
333 network.set_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER)
334
335 if row.is_anycast:
336 network.set_flag(location.NETWORK_FLAG_ANYCAST)
337
338 # Add all countries
339 log.info("Writing countries...")
340 rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
341
342 for row in rows:
343 c = writer.add_country(row.country_code)
344 c.continent_code = row.continent_code
345 c.name = row.name
346
347 # Write everything to file
348 log.info("Writing database to file...")
349 for file in ns.file:
350 writer.write(file)
351
352 def handle_update_whois(self, ns):
353 downloader = location.importer.Downloader()
354
355 # Download all sources
356 with self.db.transaction():
357 # Create some temporary tables to store parsed data
358 self.db.execute("""
359 CREATE TEMPORARY TABLE _autnums(number integer, organization text)
360 ON COMMIT DROP;
361 CREATE UNIQUE INDEX _autnums_number ON _autnums(number);
362
363 CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
364 ON COMMIT DROP;
365 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
366 """)
367
368 for source in location.importer.WHOIS_SOURCES:
369 with downloader.request(source, return_blocks=True) as f:
370 for block in f:
371 self._parse_block(block)
372
373 self.db.execute("""
374 INSERT INTO autnums(number, name)
375 SELECT _autnums.number, _organizations.name FROM _autnums
376 JOIN _organizations ON _autnums.organization = _organizations.handle
377 ON CONFLICT (number) DO UPDATE SET name = excluded.name
378 """)
379
380 self.db.execute("""
381 --- Purge any redundant entries
382 CREATE TEMPORARY TABLE _garbage ON COMMIT DROP
383 AS
384 SELECT network FROM networks candidates
385 WHERE EXISTS (
386 SELECT FROM networks
387 WHERE
388 networks.network << candidates.network
389 AND
390 networks.country = candidates.country
391 );
392
393 CREATE UNIQUE INDEX _garbage_search ON _garbage USING BTREE(network);
394
395 DELETE FROM networks WHERE EXISTS (
396 SELECT FROM _garbage WHERE networks.network = _garbage.network
397 );
398 """)
399
400 # Download all extended sources
401 for source in location.importer.EXTENDED_SOURCES:
402 with self.db.transaction():
403 # Download data
404 with downloader.request(source) as f:
405 for line in f:
406 self._parse_line(line)
407
408 def _parse_block(self, block):
409 # Get first line to find out what type of block this is
410 line = block[0]
411
412 # aut-num
413 if line.startswith("aut-num:"):
414 return self._parse_autnum_block(block)
415
416 # inetnum
417 if line.startswith("inet6num:") or line.startswith("inetnum:"):
418 return self._parse_inetnum_block(block)
419
420 # organisation
421 elif line.startswith("organisation:"):
422 return self._parse_org_block(block)
423
424 def _parse_autnum_block(self, block):
425 autnum = {}
426 for line in block:
427 # Split line
428 key, val = split_line(line)
429
430 if key == "aut-num":
431 m = re.match(r"^(AS|as)(\d+)", val)
432 if m:
433 autnum["asn"] = m.group(2)
434
435 elif key == "org":
436 autnum[key] = val
437
438 # Skip empty objects
439 if not autnum:
440 return
441
442 # Insert into database
443 self.db.execute("INSERT INTO _autnums(number, organization) \
444 VALUES(%s, %s) ON CONFLICT (number) DO UPDATE SET \
445 organization = excluded.organization",
446 autnum.get("asn"), autnum.get("org"),
447 )
448
449 def _parse_inetnum_block(self, block):
450 logging.debug("Parsing inetnum block:")
451
452 inetnum = {}
453 for line in block:
454 logging.debug(line)
455
456 # Split line
457 key, val = split_line(line)
458
459 if key == "inetnum":
460 start_address, delim, end_address = val.partition("-")
461
462 # Strip any excess space
463 start_address, end_address = start_address.rstrip(), end_address.strip()
464
465 # Convert to IP address
466 try:
467 start_address = ipaddress.ip_address(start_address)
468 end_address = ipaddress.ip_address(end_address)
469 except ValueError:
470 logging.warning("Could not parse line: %s" % line)
471 return
472
473 # Set prefix to default
474 prefix = 32
475
476 # Count number of addresses in this subnet
477 num_addresses = int(end_address) - int(start_address)
478 if num_addresses:
479 prefix -= math.log(num_addresses, 2)
480
481 inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
482
483 elif key == "inet6num":
484 inetnum[key] = val
485
486 elif key == "country":
487 if val == "UNITED STATES":
488 val = "US"
489
490 inetnum[key] = val.upper()
491
492 # Skip empty objects
493 if not inetnum:
494 return
495
496 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
497
498 # Bail out in case we have processed a non-public IP network
499 if network.is_private:
500 logging.warning("Skipping non-globally routable network: %s" % network)
501 return
502
503 self.db.execute("INSERT INTO networks(network, country) \
504 VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
505 "%s" % network, inetnum.get("country"),
506 )
507
508 def _parse_org_block(self, block):
509 org = {}
510 for line in block:
511 # Split line
512 key, val = split_line(line)
513
514 if key in ("organisation", "org-name"):
515 org[key] = val
516
517 # Skip empty objects
518 if not org:
519 return
520
521 self.db.execute("INSERT INTO _organizations(handle, name) \
522 VALUES(%s, %s) ON CONFLICT (handle) DO \
523 UPDATE SET name = excluded.name",
524 org.get("organisation"), org.get("org-name"),
525 )
526
527 def _parse_line(self, line):
528 # Skip version line
529 if line.startswith("2"):
530 return
531
532 # Skip comments
533 if line.startswith("#"):
534 return
535
536 try:
537 registry, country_code, type, line = line.split("|", 3)
538 except:
539 log.warning("Could not parse line: %s" % line)
540 return
541
542 # Skip any lines that are for stats only
543 if country_code == "*":
544 return
545
546 if type in ("ipv6", "ipv4"):
547 return self._parse_ip_line(country_code, type, line)
548
549 def _parse_ip_line(self, country, type, line):
550 try:
551 address, prefix, date, status, organization = line.split("|")
552 except ValueError:
553 organization = None
554
555 # Try parsing the line without organization
556 try:
557 address, prefix, date, status = line.split("|")
558 except ValueError:
559 log.warning("Unhandled line format: %s" % line)
560 return
561
562 # Skip anything that isn't properly assigned
563 if not status in ("assigned", "allocated"):
564 return
565
566 # Cast prefix into an integer
567 try:
568 prefix = int(prefix)
569 except:
570 log.warning("Invalid prefix: %s" % prefix)
571 return
572
573 # Fix prefix length for IPv4
574 if type == "ipv4":
575 prefix = 32 - int(math.log(prefix, 2))
576
577 # Try to parse the address
578 try:
579 network = ipaddress.ip_network("%s/%s" % (address, prefix), strict=False)
580 except ValueError:
581 log.warning("Invalid IP address: %s" % address)
582 return
583
584 self.db.execute("INSERT INTO networks(network, country) \
585 VALUES(%s, %s) ON CONFLICT (network) DO \
586 UPDATE SET country = excluded.country",
587 "%s" % network, country,
588 )
589
590 def handle_update_announcements(self, ns):
591 server = ns.server[0]
592
593 with self.db.transaction():
594 if server.startswith("/"):
595 self._handle_update_announcements_from_bird(server)
596 else:
597 self._handle_update_announcements_from_telnet(server)
598
599 # Purge anything we never want here
600 self.db.execute("""
601 -- Delete default routes
602 DELETE FROM announcements WHERE network = '::/0' OR network = '0.0.0.0/0';
603
604 -- Delete anything that is not global unicast address space
605 DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3';
606
607 -- DELETE "current network" address space
608 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '0.0.0.0/8';
609
610 -- DELETE local loopback address space
611 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '127.0.0.0/8';
612
613 -- DELETE RFC 1918 address space
614 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8';
615 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12';
616 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16';
617
618 -- DELETE test, benchmark and documentation address space
619 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.0.0/24';
620 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.2.0/24';
621 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.18.0.0/15';
622 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.51.100.0/24';
623 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '203.0.113.0/24';
624
625 -- DELETE CGNAT address space (RFC 6598)
626 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '100.64.0.0/10';
627
628 -- DELETE link local address space
629 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '169.254.0.0/16';
630
631 -- DELETE IPv6 to IPv4 (6to4) address space
632 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.88.99.0/24';
633
634 -- DELETE multicast and reserved address space
635 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '224.0.0.0/4';
636 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '240.0.0.0/4';
637
638 -- Delete networks that are too small to be in the global routing table
639 DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48;
640 DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24;
641
642 -- Delete any non-public or reserved ASNs
643 DELETE FROM announcements WHERE NOT (
644 (autnum >= 1 AND autnum <= 23455)
645 OR
646 (autnum >= 23457 AND autnum <= 64495)
647 OR
648 (autnum >= 131072 AND autnum <= 4199999999)
649 );
650
651 -- Delete everything that we have not seen for 14 days
652 DELETE FROM announcements WHERE last_seen_at <= CURRENT_TIMESTAMP - INTERVAL '14 days';
653 """)
654
655 def _handle_update_announcements_from_bird(self, server):
656 # Pre-compile the regular expression for faster searching
657 route = re.compile(b"^\s(.+?)\s+.+?\[AS(.*?).\]$")
658
659 log.info("Requesting routing table from Bird (%s)" % server)
660
661 # Send command to list all routes
662 for line in self._bird_cmd(server, "show route"):
663 m = route.match(line)
664 if not m:
665 log.debug("Could not parse line: %s" % line.decode())
666 continue
667
668 # Fetch the extracted network and ASN
669 network, autnum = m.groups()
670
671 # Insert it into the database
672 self.db.execute("INSERT INTO announcements(network, autnum) \
673 VALUES(%s, %s) ON CONFLICT (network) DO \
674 UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
675 network.decode(), autnum.decode(),
676 )
677
678 def _handle_update_announcements_from_telnet(self, server):
679 # Pre-compile regular expression for routes
680 route = re.compile(b"^\*[\s\>]i([^\s]+).+?(\d+)\si\r\n", re.MULTILINE|re.DOTALL)
681
682 with telnetlib.Telnet(server) as t:
683 # Enable debug mode
684 #if ns.debug:
685 # t.set_debuglevel(10)
686
687 # Wait for console greeting
688 greeting = t.read_until(b"> ", timeout=30)
689 if not greeting:
690 log.error("Could not get a console prompt")
691 return 1
692
693 # Disable pagination
694 t.write(b"terminal length 0\n")
695
696 # Wait for the prompt to return
697 t.read_until(b"> ")
698
699 # Fetch the routing tables
700 for protocol in ("ipv6", "ipv4"):
701 log.info("Requesting %s routing table" % protocol)
702
703 # Request the full unicast routing table
704 t.write(b"show bgp %s unicast\n" % protocol.encode())
705
706 # Read entire header which ends with "Path"
707 t.read_until(b"Path\r\n")
708
709 while True:
710 # Try reading a full entry
711 # Those might be broken across multiple lines but ends with i
712 line = t.read_until(b"i\r\n", timeout=5)
713 if not line:
714 break
715
716 # Show line for debugging
717 #log.debug(repr(line))
718
719 # Try finding a route in here
720 m = route.match(line)
721 if m:
722 network, autnum = m.groups()
723
724 # Convert network to string
725 network = network.decode()
726
727 # Append /24 for IPv4 addresses
728 if not "/" in network and not ":" in network:
729 network = "%s/24" % network
730
731 # Convert AS number to integer
732 autnum = int(autnum)
733
734 log.info("Found announcement for %s by %s" % (network, autnum))
735
736 self.db.execute("INSERT INTO announcements(network, autnum) \
737 VALUES(%s, %s) ON CONFLICT (network) DO \
738 UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
739 network, autnum,
740 )
741
742 log.info("Finished reading the %s routing table" % protocol)
743
744 def _bird_cmd(self, socket_path, command):
745 # Connect to the socket
746 s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
747 s.connect(socket_path)
748
749 # Allocate some buffer
750 buffer = b""
751
752 # Send the command
753 s.send(b"%s\n" % command.encode())
754
755 while True:
756 # Fill up the buffer
757 buffer += s.recv(4096)
758
759 while True:
760 # Search for the next newline
761 pos = buffer.find(b"\n")
762
763 # If we cannot find one, we go back and read more data
764 if pos <= 0:
765 break
766
767 # Cut after the newline character
768 pos += 1
769
770 # Split the line we want and keep the rest in buffer
771 line, buffer = buffer[:pos], buffer[pos:]
772
773 # Look for the end-of-output indicator
774 if line == b"0000 \n":
775 return
776
777 # Otherwise return the line
778 yield line
779
780 def handle_update_overrides(self, ns):
781 with self.db.transaction():
782 # Drop all data that we have
783 self.db.execute("""
784 TRUNCATE TABLE autnum_overrides;
785 TRUNCATE TABLE network_overrides;
786 """)
787
788 for file in ns.files:
789 log.info("Reading %s..." % file)
790
791 with open(file, "rb") as f:
792 for type, block in location.importer.read_blocks(f):
793 if type == "net":
794 network = block.get("net")
795 # Try to parse and normalise the network
796 try:
797 network = ipaddress.ip_network(network, strict=False)
798 except ValueError as e:
799 log.warning("Invalid IP network: %s: %s" % (network, e))
800 continue
801
802 # Prevent that we overwrite all networks
803 if network.prefixlen == 0:
804 log.warning("Skipping %s: You cannot overwrite default" % network)
805 continue
806
807 self.db.execute("""
808 INSERT INTO network_overrides(
809 network,
810 country,
811 is_anonymous_proxy,
812 is_satellite_provider,
813 is_anycast
814 ) VALUES (%s, %s, %s, %s, %s)
815 ON CONFLICT (network) DO NOTHING""",
816 "%s" % network,
817 block.get("country"),
818 self._parse_bool(block, "is-anonymous-proxy"),
819 self._parse_bool(block, "is-satellite-provider"),
820 self._parse_bool(block, "is-anycast"),
821 )
822
823 elif type == "aut-num":
824 autnum = block.get("aut-num")
825
826 # Check if AS number begins with "AS"
827 if not autnum.startswith("AS"):
828 log.warning("Invalid AS number: %s" % autnum)
829 continue
830
831 # Strip "AS"
832 autnum = autnum[2:]
833
834 self.db.execute("""
835 INSERT INTO autnum_overrides(
836 number,
837 name,
838 country,
839 is_anonymous_proxy,
840 is_satellite_provider,
841 is_anycast
842 ) VALUES(%s, %s, %s, %s, %s, %s)
843 ON CONFLICT DO NOTHING""",
844 autnum,
845 block.get("name"),
846 block.get("country"),
847 self._parse_bool(block, "is-anonymous-proxy"),
848 self._parse_bool(block, "is-satellite-provider"),
849 self._parse_bool(block, "is-anycast"),
850 )
851
852 else:
853 log.warning("Unsupport type: %s" % type)
854
855 @staticmethod
856 def _parse_bool(block, key):
857 val = block.get(key)
858
859 # There is no point to proceed when we got None
860 if val is None:
861 return
862
863 # Convert to lowercase
864 val = val.lower()
865
866 # True
867 if val in ("yes", "1"):
868 return True
869
870 # False
871 if val in ("no", "0"):
872 return False
873
874 # Default to None
875 return None
876
877 def handle_import_countries(self, ns):
878 with self.db.transaction():
879 # Drop all data that we have
880 self.db.execute("TRUNCATE TABLE countries")
881
882 for file in ns.file:
883 for line in file:
884 line = line.rstrip()
885
886 # Ignore any comments
887 if line.startswith("#"):
888 continue
889
890 try:
891 country_code, continent_code, name = line.split(maxsplit=2)
892 except:
893 log.warning("Could not parse line: %s" % line)
894 continue
895
896 self.db.execute("INSERT INTO countries(country_code, name, continent_code) \
897 VALUES(%s, %s, %s) ON CONFLICT DO NOTHING", country_code, name, continent_code)
898
899
900 def split_line(line):
901 key, colon, val = line.partition(":")
902
903 # Strip any excess space
904 key = key.strip()
905 val = val.strip()
906
907 return key, val
908
909 def main():
910 # Run the command line interface
911 c = CLI()
912 c.run()
913
914 main()
A library to determine the location of someone on the Internet