]> git.ipfire.org Git - people/ms/libloc.git/blob - src/python/location-importer.in
projects / people / ms / libloc.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
location-importer.in: only import relevant data from AFRINIC, APNIC and RIPE
[people/ms/libloc.git] / src / python / location-importer.in
1 #!/usr/bin/python3
2 ###############################################################################
3 # #
4 # libloc - A library to determine the location of someone on the Internet #
5 # #
6 # Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
7 # #
8 # This library is free software; you can redistribute it and/or #
9 # modify it under the terms of the GNU Lesser General Public #
10 # License as published by the Free Software Foundation; either #
11 # version 2.1 of the License, or (at your option) any later version. #
12 # #
13 # This library is distributed in the hope that it will be useful, #
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
16 # Lesser General Public License for more details. #
17 # #
18 ###############################################################################
19
20 import argparse
21 import ipaddress
22 import logging
23 import math
24 import re
25 import socket
26 import sys
27 import telnetlib
28
29 # Load our location module
30 import location
31 import location.database
32 import location.importer
33 from location.i18n import _
34
35 # Initialise logging
36 log = logging.getLogger("location.importer")
37 log.propagate = 1
38
39 class CLI(object):
40 def parse_cli(self):
41 parser = argparse.ArgumentParser(
42 description=_("Location Importer Command Line Interface"),
43 )
44 subparsers = parser.add_subparsers()
45
46 # Global configuration flags
47 parser.add_argument("--debug", action="store_true",
48 help=_("Enable debug output"))
49 parser.add_argument("--quiet", action="store_true",
50 help=_("Enable quiet mode"))
51
52 # version
53 parser.add_argument("--version", action="version",
54 version="%(prog)s @VERSION@")
55
56 # Database
57 parser.add_argument("--database-host", required=True,
58 help=_("Database Hostname"), metavar=_("HOST"))
59 parser.add_argument("--database-name", required=True,
60 help=_("Database Name"), metavar=_("NAME"))
61 parser.add_argument("--database-username", required=True,
62 help=_("Database Username"), metavar=_("USERNAME"))
63 parser.add_argument("--database-password", required=True,
64 help=_("Database Password"), metavar=_("PASSWORD"))
65
66 # Write Database
67 write = subparsers.add_parser("write", help=_("Write database to file"))
68 write.set_defaults(func=self.handle_write)
69 write.add_argument("file", nargs=1, help=_("Database File"))
70 write.add_argument("--signing-key", nargs="?", type=open, help=_("Signing Key"))
71 write.add_argument("--backup-signing-key", nargs="?", type=open, help=_("Backup Signing Key"))
72 write.add_argument("--vendor", nargs="?", help=_("Sets the vendor"))
73 write.add_argument("--description", nargs="?", help=_("Sets a description"))
74 write.add_argument("--license", nargs="?", help=_("Sets the license"))
75 write.add_argument("--version", type=int, help=_("Database Format Version"))
76
77 # Update WHOIS
78 update_whois = subparsers.add_parser("update-whois", help=_("Update WHOIS Information"))
79 update_whois.set_defaults(func=self.handle_update_whois)
80
81 # Update announcements
82 update_announcements = subparsers.add_parser("update-announcements",
83 help=_("Update BGP Annoucements"))
84 update_announcements.set_defaults(func=self.handle_update_announcements)
85 update_announcements.add_argument("server", nargs=1,
86 help=_("Route Server to connect to"), metavar=_("SERVER"))
87
88 # Update overrides
89 update_overrides = subparsers.add_parser("update-overrides",
90 help=_("Update overrides"),
91 )
92 update_overrides.add_argument(
93 "files", nargs="+", help=_("Files to import"),
94 )
95 update_overrides.set_defaults(func=self.handle_update_overrides)
96
97 # Import countries
98 import_countries = subparsers.add_parser("import-countries",
99 help=_("Import countries"),
100 )
101 import_countries.add_argument("file", nargs=1, type=argparse.FileType("r"),
102 help=_("File to import"))
103 import_countries.set_defaults(func=self.handle_import_countries)
104
105 args = parser.parse_args()
106
107 # Configure logging
108 if args.debug:
109 location.logger.set_level(logging.DEBUG)
110 elif args.quiet:
111 location.logger.set_level(logging.WARNING)
112
113 # Print usage if no action was given
114 if not "func" in args:
115 parser.print_usage()
116 sys.exit(2)
117
118 return args
119
120 def run(self):
121 # Parse command line arguments
122 args = self.parse_cli()
123
124 # Initialise database
125 self.db = self._setup_database(args)
126
127 # Call function
128 ret = args.func(args)
129
130 # Return with exit code
131 if ret:
132 sys.exit(ret)
133
134 # Otherwise just exit
135 sys.exit(0)
136
137 def _setup_database(self, ns):
138 """
139 Initialise the database
140 """
141 # Connect to database
142 db = location.database.Connection(
143 host=ns.database_host, database=ns.database_name,
144 user=ns.database_username, password=ns.database_password,
145 )
146
147 with db.transaction():
148 db.execute("""
149 -- announcements
150 CREATE TABLE IF NOT EXISTS announcements(network inet, autnum bigint,
151 first_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP,
152 last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
153 CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
154 CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
155
156 -- autnums
157 CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
158 CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number);
159
160 -- countries
161 CREATE TABLE IF NOT EXISTS countries(
162 country_code text NOT NULL, name text NOT NULL, continent_code text NOT NULL);
163 CREATE UNIQUE INDEX IF NOT EXISTS countries_country_code ON countries(country_code);
164
165 -- networks
166 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
167 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
168 CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
169 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
170
171 -- overrides
172 CREATE TABLE IF NOT EXISTS autnum_overrides(
173 number bigint NOT NULL,
174 name text,
175 country text,
176 is_anonymous_proxy boolean,
177 is_satellite_provider boolean,
178 is_anycast boolean
179 );
180 CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number
181 ON autnum_overrides(number);
182
183 CREATE TABLE IF NOT EXISTS network_overrides(
184 network inet NOT NULL,
185 country text,
186 is_anonymous_proxy boolean,
187 is_satellite_provider boolean,
188 is_anycast boolean
189 );
190 CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
191 ON network_overrides(network);
192 """)
193
194 return db
195
196 def handle_write(self, ns):
197 """
198 Compiles a database in libloc format out of what is in the database
199 """
200 # Allocate a writer
201 writer = location.Writer(ns.signing_key, ns.backup_signing_key)
202
203 # Set all metadata
204 if ns.vendor:
205 writer.vendor = ns.vendor
206
207 if ns.description:
208 writer.description = ns.description
209
210 if ns.license:
211 writer.license = ns.license
212
213 # Add all Autonomous Systems
214 log.info("Writing Autonomous Systems...")
215
216 # Select all ASes with a name
217 rows = self.db.query("""
218 SELECT
219 autnums.number AS number,
220 COALESCE(
221 (SELECT overrides.name FROM autnum_overrides overrides
222 WHERE overrides.number = autnums.number),
223 autnums.name
224 ) AS name
225 FROM autnums
226 WHERE name <> %s ORDER BY number
227 """, "")
228
229 for row in rows:
230 a = writer.add_as(row.number)
231 a.name = row.name
232
233 # Add all networks
234 log.info("Writing networks...")
235
236 # Select all known networks
237 rows = self.db.query("""
238 -- Get a (sorted) list of all known networks
239 WITH known_networks AS (
240 SELECT network FROM announcements
241 UNION
242 SELECT network FROM networks
243 ORDER BY network
244 )
245
246 -- Return a list of those networks enriched with all
247 -- other information that we store in the database
248 SELECT
249 DISTINCT ON (known_networks.network)
250 known_networks.network AS network,
251 announcements.autnum AS autnum,
252
253 -- Country
254 COALESCE(
255 (
256 SELECT country FROM network_overrides overrides
257 WHERE announcements.network <<= overrides.network
258 ORDER BY masklen(overrides.network) DESC
259 LIMIT 1
260 ),
261 (
262 SELECT country FROM autnum_overrides overrides
263 WHERE announcements.autnum = overrides.number
264 ),
265 networks.country
266 ) AS country,
267
268 -- Flags
269 COALESCE(
270 (
271 SELECT is_anonymous_proxy FROM network_overrides overrides
272 WHERE announcements.network <<= overrides.network
273 ORDER BY masklen(overrides.network) DESC
274 LIMIT 1
275 ),
276 (
277 SELECT is_anonymous_proxy FROM autnum_overrides overrides
278 WHERE announcements.autnum = overrides.number
279 ),
280 FALSE
281 ) AS is_anonymous_proxy,
282 COALESCE(
283 (
284 SELECT is_satellite_provider FROM network_overrides overrides
285 WHERE announcements.network <<= overrides.network
286 ORDER BY masklen(overrides.network) DESC
287 LIMIT 1
288 ),
289 (
290 SELECT is_satellite_provider FROM autnum_overrides overrides
291 WHERE announcements.autnum = overrides.number
292 ),
293 FALSE
294 ) AS is_satellite_provider,
295 COALESCE(
296 (
297 SELECT is_anycast FROM network_overrides overrides
298 WHERE announcements.network <<= overrides.network
299 ORDER BY masklen(overrides.network) DESC
300 LIMIT 1
301 ),
302 (
303 SELECT is_anycast FROM autnum_overrides overrides
304 WHERE announcements.autnum = overrides.number
305 ),
306 FALSE
307 ) AS is_anycast,
308
309 -- Must be part of returned values for ORDER BY clause
310 masklen(announcements.network) AS sort_a,
311 masklen(networks.network) AS sort_b
312 FROM known_networks
313 LEFT JOIN announcements ON known_networks.network <<= announcements.network
314 LEFT JOIN networks ON known_networks.network <<= networks.network
315 ORDER BY known_networks.network, sort_a DESC, sort_b DESC
316 """)
317
318 for row in rows:
319 network = writer.add_network(row.network)
320
321 # Save country
322 if row.country:
323 network.country_code = row.country
324
325 # Save ASN
326 if row.autnum:
327 network.asn = row.autnum
328
329 # Set flags
330 if row.is_anonymous_proxy:
331 network.set_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY)
332
333 if row.is_satellite_provider:
334 network.set_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER)
335
336 if row.is_anycast:
337 network.set_flag(location.NETWORK_FLAG_ANYCAST)
338
339 # Add all countries
340 log.info("Writing countries...")
341 rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
342
343 for row in rows:
344 c = writer.add_country(row.country_code)
345 c.continent_code = row.continent_code
346 c.name = row.name
347
348 # Write everything to file
349 log.info("Writing database to file...")
350 for file in ns.file:
351 writer.write(file)
352
353 def handle_update_whois(self, ns):
354 downloader = location.importer.Downloader()
355
356 # Download all sources
357 with self.db.transaction():
358 # Create some temporary tables to store parsed data
359 self.db.execute("""
360 CREATE TEMPORARY TABLE _autnums(number integer, organization text)
361 ON COMMIT DROP;
362 CREATE UNIQUE INDEX _autnums_number ON _autnums(number);
363
364 CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
365 ON COMMIT DROP;
366 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
367
368 CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
369 ON COMMIT DROP;
370 CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
371 CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
372 """)
373
374 # Remove all previously imported content
375 self.db.execute("""
376 TRUNCATE TABLE networks;
377 """)
378
379 for source in location.importer.WHOIS_SOURCES:
380 with downloader.request(source, return_blocks=True) as f:
381 for block in f:
382 self._parse_block(block)
383
384 # Process all parsed networks from every RIR we happen to have access to,
385 # insert the largest network chunks into the networks table immediately...
386 families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
387
388 for family in (row.family for row in families):
389 smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
390
391 self.db.execute("INSERT INTO networks(network, country) \
392 SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
393
394 # ... determine any other prefixes for this network family, ...
395 prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
396 WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
397
398 # ... and insert networks with this prefix in case they provide additional
399 # information (i. e. subnet of a larger chunk with a different country)
400 for prefix in (row.prefix for row in prefixes):
401 self.db.execute("""
402 WITH candidates AS (
403 SELECT
404 _rirdata.network,
405 _rirdata.country
406 FROM
407 _rirdata
408 WHERE
409 family(_rirdata.network) = %s
410 AND
411 masklen(_rirdata.network) = %s
412 ),
413 filtered AS (
414 SELECT
415 DISTINCT ON (c.network)
416 c.network,
417 c.country,
418 masklen(networks.network),
419 networks.country AS parent_country
420 FROM
421 candidates c
422 LEFT JOIN
423 networks
424 ON
425 c.network << networks.network
426 ORDER BY
427 c.network,
428 masklen(networks.network) DESC NULLS LAST
429 )
430 INSERT INTO
431 networks(network, country)
432 SELECT
433 network,
434 country
435 FROM
436 filtered
437 WHERE
438 parent_country IS NULL
439 OR
440 country <> parent_country
441 ON CONFLICT DO NOTHING""",
442 family, prefix,
443 )
444
445 self.db.execute("""
446 INSERT INTO autnums(number, name)
447 SELECT _autnums.number, _organizations.name FROM _autnums
448 JOIN _organizations ON _autnums.organization = _organizations.handle
449 ON CONFLICT (number) DO UPDATE SET name = excluded.name;
450 """)
451
452 # Download all extended sources
453 for source in location.importer.EXTENDED_SOURCES:
454 with self.db.transaction():
455 # Download data
456 with downloader.request(source) as f:
457 for line in f:
458 self._parse_line(line)
459
460 def _parse_block(self, block):
461 # Get first line to find out what type of block this is
462 line = block[0]
463
464 # aut-num
465 if line.startswith("aut-num:"):
466 return self._parse_autnum_block(block)
467
468 # inetnum
469 if line.startswith("inet6num:") or line.startswith("inetnum:"):
470 return self._parse_inetnum_block(block)
471
472 # organisation
473 elif line.startswith("organisation:"):
474 return self._parse_org_block(block)
475
476 def _parse_autnum_block(self, block):
477 autnum = {}
478 for line in block:
479 # Split line
480 key, val = split_line(line)
481
482 if key == "aut-num":
483 m = re.match(r"^(AS|as)(\d+)", val)
484 if m:
485 autnum["asn"] = m.group(2)
486
487 elif key == "org":
488 autnum[key] = val
489
490 # Skip empty objects
491 if not autnum:
492 return
493
494 # Insert into database
495 self.db.execute("INSERT INTO _autnums(number, organization) \
496 VALUES(%s, %s) ON CONFLICT (number) DO UPDATE SET \
497 organization = excluded.organization",
498 autnum.get("asn"), autnum.get("org"),
499 )
500
501 def _parse_inetnum_block(self, block):
502 logging.debug("Parsing inetnum block:")
503
504 inetnum = {}
505 for line in block:
506 logging.debug(line)
507
508 # Split line
509 key, val = split_line(line)
510
511 if key == "inetnum":
512 start_address, delim, end_address = val.partition("-")
513
514 # Strip any excess space
515 start_address, end_address = start_address.rstrip(), end_address.strip()
516
517 # Convert to IP address
518 try:
519 start_address = ipaddress.ip_address(start_address)
520 end_address = ipaddress.ip_address(end_address)
521 except ValueError:
522 logging.warning("Could not parse line: %s" % line)
523 return
524
525 # Set prefix to default
526 prefix = 32
527
528 # Count number of addresses in this subnet
529 num_addresses = int(end_address) - int(start_address)
530 if num_addresses:
531 prefix -= math.log(num_addresses, 2)
532
533 inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
534
535 elif key == "inet6num":
536 inetnum[key] = val
537
538 elif key == "country":
539 if val == "UNITED STATES":
540 val = "US"
541
542 inetnum[key] = val.upper()
543
544 # Skip empty objects
545 if not inetnum or not "country" in inetnum:
546 return
547
548 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
549
550 # Bail out in case we have processed a network covering the entire IP range, which
551 # is necessary to work around faulty (?) IPv6 network processing
552 if network.prefixlen == 0:
553 logging.warning("Skipping network covering the entire IP adress range: %s" % network)
554 return
555
556 # Bail out in case we have processed a network whose prefix length indicates it is
557 # not globally routable (we have decided not to process them at the moment, as they
558 # significantly enlarge our database without providing very helpful additional information)
559 if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
560 logging.info("Skipping network too small to be publicly announced: %s" % network)
561 return
562
563 # Bail out in case we have processed a non-public IP network
564 if network.is_private:
565 logging.warning("Skipping non-globally routable network: %s" % network)
566 return
567
568 self.db.execute("INSERT INTO _rirdata(network, country) \
569 VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
570 "%s" % network, inetnum.get("country"),
571 )
572
573 def _parse_org_block(self, block):
574 org = {}
575 for line in block:
576 # Split line
577 key, val = split_line(line)
578
579 if key in ("organisation", "org-name"):
580 org[key] = val
581
582 # Skip empty objects
583 if not org:
584 return
585
586 self.db.execute("INSERT INTO _organizations(handle, name) \
587 VALUES(%s, %s) ON CONFLICT (handle) DO \
588 UPDATE SET name = excluded.name",
589 org.get("organisation"), org.get("org-name"),
590 )
591
592 def _parse_line(self, line):
593 # Skip version line
594 if line.startswith("2"):
595 return
596
597 # Skip comments
598 if line.startswith("#"):
599 return
600
601 try:
602 registry, country_code, type, line = line.split("|", 3)
603 except:
604 log.warning("Could not parse line: %s" % line)
605 return
606
607 # Skip any lines that are for stats only
608 if country_code == "*":
609 return
610
611 if type in ("ipv6", "ipv4"):
612 return self._parse_ip_line(country_code, type, line)
613
614 def _parse_ip_line(self, country, type, line):
615 try:
616 address, prefix, date, status, organization = line.split("|")
617 except ValueError:
618 organization = None
619
620 # Try parsing the line without organization
621 try:
622 address, prefix, date, status = line.split("|")
623 except ValueError:
624 log.warning("Unhandled line format: %s" % line)
625 return
626
627 # Skip anything that isn't properly assigned
628 if not status in ("assigned", "allocated"):
629 return
630
631 # Cast prefix into an integer
632 try:
633 prefix = int(prefix)
634 except:
635 log.warning("Invalid prefix: %s" % prefix)
636 return
637
638 # Fix prefix length for IPv4
639 if type == "ipv4":
640 prefix = 32 - int(math.log(prefix, 2))
641
642 # Try to parse the address
643 try:
644 network = ipaddress.ip_network("%s/%s" % (address, prefix), strict=False)
645 except ValueError:
646 log.warning("Invalid IP address: %s" % address)
647 return
648
649 self.db.execute("INSERT INTO networks(network, country) \
650 VALUES(%s, %s) ON CONFLICT (network) DO \
651 UPDATE SET country = excluded.country",
652 "%s" % network, country,
653 )
654
655 def handle_update_announcements(self, ns):
656 server = ns.server[0]
657
658 with self.db.transaction():
659 if server.startswith("/"):
660 self._handle_update_announcements_from_bird(server)
661 else:
662 self._handle_update_announcements_from_telnet(server)
663
664 # Purge anything we never want here
665 self.db.execute("""
666 -- Delete default routes
667 DELETE FROM announcements WHERE network = '::/0' OR network = '0.0.0.0/0';
668
669 -- Delete anything that is not global unicast address space
670 DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3';
671
672 -- DELETE "current network" address space
673 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '0.0.0.0/8';
674
675 -- DELETE local loopback address space
676 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '127.0.0.0/8';
677
678 -- DELETE RFC 1918 address space
679 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8';
680 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12';
681 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16';
682
683 -- DELETE test, benchmark and documentation address space
684 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.0.0/24';
685 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.2.0/24';
686 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.18.0.0/15';
687 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.51.100.0/24';
688 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '203.0.113.0/24';
689
690 -- DELETE CGNAT address space (RFC 6598)
691 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '100.64.0.0/10';
692
693 -- DELETE link local address space
694 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '169.254.0.0/16';
695
696 -- DELETE IPv6 to IPv4 (6to4) address space
697 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.88.99.0/24';
698
699 -- DELETE multicast and reserved address space
700 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '224.0.0.0/4';
701 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '240.0.0.0/4';
702
703 -- Delete networks that are too small to be in the global routing table
704 DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48;
705 DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24;
706
707 -- Delete any non-public or reserved ASNs
708 DELETE FROM announcements WHERE NOT (
709 (autnum >= 1 AND autnum <= 23455)
710 OR
711 (autnum >= 23457 AND autnum <= 64495)
712 OR
713 (autnum >= 131072 AND autnum <= 4199999999)
714 );
715
716 -- Delete everything that we have not seen for 14 days
717 DELETE FROM announcements WHERE last_seen_at <= CURRENT_TIMESTAMP - INTERVAL '14 days';
718 """)
719
720 def _handle_update_announcements_from_bird(self, server):
721 # Pre-compile the regular expression for faster searching
722 route = re.compile(b"^\s(.+?)\s+.+?\[AS(.*?).\]$")
723
724 log.info("Requesting routing table from Bird (%s)" % server)
725
726 # Send command to list all routes
727 for line in self._bird_cmd(server, "show route"):
728 m = route.match(line)
729 if not m:
730 log.debug("Could not parse line: %s" % line.decode())
731 continue
732
733 # Fetch the extracted network and ASN
734 network, autnum = m.groups()
735
736 # Insert it into the database
737 self.db.execute("INSERT INTO announcements(network, autnum) \
738 VALUES(%s, %s) ON CONFLICT (network) DO \
739 UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
740 network.decode(), autnum.decode(),
741 )
742
743 def _handle_update_announcements_from_telnet(self, server):
744 # Pre-compile regular expression for routes
745 route = re.compile(b"^\*[\s\>]i([^\s]+).+?(\d+)\si\r\n", re.MULTILINE|re.DOTALL)
746
747 with telnetlib.Telnet(server) as t:
748 # Enable debug mode
749 #if ns.debug:
750 # t.set_debuglevel(10)
751
752 # Wait for console greeting
753 greeting = t.read_until(b"> ", timeout=30)
754 if not greeting:
755 log.error("Could not get a console prompt")
756 return 1
757
758 # Disable pagination
759 t.write(b"terminal length 0\n")
760
761 # Wait for the prompt to return
762 t.read_until(b"> ")
763
764 # Fetch the routing tables
765 for protocol in ("ipv6", "ipv4"):
766 log.info("Requesting %s routing table" % protocol)
767
768 # Request the full unicast routing table
769 t.write(b"show bgp %s unicast\n" % protocol.encode())
770
771 # Read entire header which ends with "Path"
772 t.read_until(b"Path\r\n")
773
774 while True:
775 # Try reading a full entry
776 # Those might be broken across multiple lines but ends with i
777 line = t.read_until(b"i\r\n", timeout=5)
778 if not line:
779 break
780
781 # Show line for debugging
782 #log.debug(repr(line))
783
784 # Try finding a route in here
785 m = route.match(line)
786 if m:
787 network, autnum = m.groups()
788
789 # Convert network to string
790 network = network.decode()
791
792 # Append /24 for IPv4 addresses
793 if not "/" in network and not ":" in network:
794 network = "%s/24" % network
795
796 # Convert AS number to integer
797 autnum = int(autnum)
798
799 log.info("Found announcement for %s by %s" % (network, autnum))
800
801 self.db.execute("INSERT INTO announcements(network, autnum) \
802 VALUES(%s, %s) ON CONFLICT (network) DO \
803 UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
804 network, autnum,
805 )
806
807 log.info("Finished reading the %s routing table" % protocol)
808
809 def _bird_cmd(self, socket_path, command):
810 # Connect to the socket
811 s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
812 s.connect(socket_path)
813
814 # Allocate some buffer
815 buffer = b""
816
817 # Send the command
818 s.send(b"%s\n" % command.encode())
819
820 while True:
821 # Fill up the buffer
822 buffer += s.recv(4096)
823
824 while True:
825 # Search for the next newline
826 pos = buffer.find(b"\n")
827
828 # If we cannot find one, we go back and read more data
829 if pos <= 0:
830 break
831
832 # Cut after the newline character
833 pos += 1
834
835 # Split the line we want and keep the rest in buffer
836 line, buffer = buffer[:pos], buffer[pos:]
837
838 # Look for the end-of-output indicator
839 if line == b"0000 \n":
840 return
841
842 # Otherwise return the line
843 yield line
844
845 def handle_update_overrides(self, ns):
846 with self.db.transaction():
847 # Drop all data that we have
848 self.db.execute("""
849 TRUNCATE TABLE autnum_overrides;
850 TRUNCATE TABLE network_overrides;
851 """)
852
853 for file in ns.files:
854 log.info("Reading %s..." % file)
855
856 with open(file, "rb") as f:
857 for type, block in location.importer.read_blocks(f):
858 if type == "net":
859 network = block.get("net")
860 # Try to parse and normalise the network
861 try:
862 network = ipaddress.ip_network(network, strict=False)
863 except ValueError as e:
864 log.warning("Invalid IP network: %s: %s" % (network, e))
865 continue
866
867 # Prevent that we overwrite all networks
868 if network.prefixlen == 0:
869 log.warning("Skipping %s: You cannot overwrite default" % network)
870 continue
871
872 self.db.execute("""
873 INSERT INTO network_overrides(
874 network,
875 country,
876 is_anonymous_proxy,
877 is_satellite_provider,
878 is_anycast
879 ) VALUES (%s, %s, %s, %s, %s)
880 ON CONFLICT (network) DO NOTHING""",
881 "%s" % network,
882 block.get("country"),
883 self._parse_bool(block, "is-anonymous-proxy"),
884 self._parse_bool(block, "is-satellite-provider"),
885 self._parse_bool(block, "is-anycast"),
886 )
887
888 elif type == "aut-num":
889 autnum = block.get("aut-num")
890
891 # Check if AS number begins with "AS"
892 if not autnum.startswith("AS"):
893 log.warning("Invalid AS number: %s" % autnum)
894 continue
895
896 # Strip "AS"
897 autnum = autnum[2:]
898
899 self.db.execute("""
900 INSERT INTO autnum_overrides(
901 number,
902 name,
903 country,
904 is_anonymous_proxy,
905 is_satellite_provider,
906 is_anycast
907 ) VALUES(%s, %s, %s, %s, %s, %s)
908 ON CONFLICT DO NOTHING""",
909 autnum,
910 block.get("name"),
911 block.get("country"),
912 self._parse_bool(block, "is-anonymous-proxy"),
913 self._parse_bool(block, "is-satellite-provider"),
914 self._parse_bool(block, "is-anycast"),
915 )
916
917 else:
918 log.warning("Unsupport type: %s" % type)
919
920 @staticmethod
921 def _parse_bool(block, key):
922 val = block.get(key)
923
924 # There is no point to proceed when we got None
925 if val is None:
926 return
927
928 # Convert to lowercase
929 val = val.lower()
930
931 # True
932 if val in ("yes", "1"):
933 return True
934
935 # False
936 if val in ("no", "0"):
937 return False
938
939 # Default to None
940 return None
941
942 def handle_import_countries(self, ns):
943 with self.db.transaction():
944 # Drop all data that we have
945 self.db.execute("TRUNCATE TABLE countries")
946
947 for file in ns.file:
948 for line in file:
949 line = line.rstrip()
950
951 # Ignore any comments
952 if line.startswith("#"):
953 continue
954
955 try:
956 country_code, continent_code, name = line.split(maxsplit=2)
957 except:
958 log.warning("Could not parse line: %s" % line)
959 continue
960
961 self.db.execute("INSERT INTO countries(country_code, name, continent_code) \
962 VALUES(%s, %s, %s) ON CONFLICT DO NOTHING", country_code, name, continent_code)
963
964
965 def split_line(line):
966 key, colon, val = line.partition(":")
967
968 # Strip any excess space
969 key = key.strip()
970 val = val.strip()
971
972 return key, val
973
974 def main():
975 # Run the command line interface
976 c = CLI()
977 c.run()
978
979 main()
A library to determine the location of someone on the Internet