]> git.ipfire.org Git - location/libloc.git/blob - src/python/location-importer.in
projects / location / libloc.git / blob
? search:


1a025bb1005976d0d5128c863ebc791b00ca076d
[location/libloc.git] / src / python / location-importer.in
1 #!/usr/bin/python3
2 ###############################################################################
3 # #
4 # libloc - A library to determine the location of someone on the Internet #
5 # #
6 # Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
7 # #
8 # This library is free software; you can redistribute it and/or #
9 # modify it under the terms of the GNU Lesser General Public #
10 # License as published by the Free Software Foundation; either #
11 # version 2.1 of the License, or (at your option) any later version. #
12 # #
13 # This library is distributed in the hope that it will be useful, #
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
16 # Lesser General Public License for more details. #
17 # #
18 ###############################################################################
19
20 import argparse
21 import ipaddress
22 import logging
23 import math
24 import re
25 import socket
26 import sys
27 import telnetlib
28
29 # Load our location module
30 import location
31 import location.database
32 import location.importer
33 from location.i18n import _
34
35 # Initialise logging
36 log = logging.getLogger("location.importer")
37 log.propagate = 1
38
39 class CLI(object):
40 def parse_cli(self):
41 parser = argparse.ArgumentParser(
42 description=_("Location Importer Command Line Interface"),
43 )
44 subparsers = parser.add_subparsers()
45
46 # Global configuration flags
47 parser.add_argument("--debug", action="store_true",
48 help=_("Enable debug output"))
49 parser.add_argument("--quiet", action="store_true",
50 help=_("Enable quiet mode"))
51
52 # version
53 parser.add_argument("--version", action="version",
54 version="%(prog)s @VERSION@")
55
56 # Database
57 parser.add_argument("--database-host", required=True,
58 help=_("Database Hostname"), metavar=_("HOST"))
59 parser.add_argument("--database-name", required=True,
60 help=_("Database Name"), metavar=_("NAME"))
61 parser.add_argument("--database-username", required=True,
62 help=_("Database Username"), metavar=_("USERNAME"))
63 parser.add_argument("--database-password", required=True,
64 help=_("Database Password"), metavar=_("PASSWORD"))
65
66 # Write Database
67 write = subparsers.add_parser("write", help=_("Write database to file"))
68 write.set_defaults(func=self.handle_write)
69 write.add_argument("file", nargs=1, help=_("Database File"))
70 write.add_argument("--signing-key", nargs="?", type=open, help=_("Signing Key"))
71 write.add_argument("--backup-signing-key", nargs="?", type=open, help=_("Backup Signing Key"))
72 write.add_argument("--vendor", nargs="?", help=_("Sets the vendor"))
73 write.add_argument("--description", nargs="?", help=_("Sets a description"))
74 write.add_argument("--license", nargs="?", help=_("Sets the license"))
75 write.add_argument("--version", type=int, help=_("Database Format Version"))
76
77 # Update WHOIS
78 update_whois = subparsers.add_parser("update-whois", help=_("Update WHOIS Information"))
79 update_whois.set_defaults(func=self.handle_update_whois)
80
81 # Update announcements
82 update_announcements = subparsers.add_parser("update-announcements",
83 help=_("Update BGP Annoucements"))
84 update_announcements.set_defaults(func=self.handle_update_announcements)
85 update_announcements.add_argument("server", nargs=1,
86 help=_("Route Server to connect to"), metavar=_("SERVER"))
87
88 # Update overrides
89 update_overrides = subparsers.add_parser("update-overrides",
90 help=_("Update overrides"),
91 )
92 update_overrides.add_argument(
93 "files", nargs="+", help=_("Files to import"),
94 )
95 update_overrides.set_defaults(func=self.handle_update_overrides)
96
97 # Import countries
98 import_countries = subparsers.add_parser("import-countries",
99 help=_("Import countries"),
100 )
101 import_countries.add_argument("file", nargs=1, type=argparse.FileType("r"),
102 help=_("File to import"))
103 import_countries.set_defaults(func=self.handle_import_countries)
104
105 args = parser.parse_args()
106
107 # Configure logging
108 if args.debug:
109 location.logger.set_level(logging.DEBUG)
110 elif args.quiet:
111 location.logger.set_level(logging.WARNING)
112
113 # Print usage if no action was given
114 if not "func" in args:
115 parser.print_usage()
116 sys.exit(2)
117
118 return args
119
120 def run(self):
121 # Parse command line arguments
122 args = self.parse_cli()
123
124 # Initialise database
125 self.db = self._setup_database(args)
126
127 # Call function
128 ret = args.func(args)
129
130 # Return with exit code
131 if ret:
132 sys.exit(ret)
133
134 # Otherwise just exit
135 sys.exit(0)
136
137 def _setup_database(self, ns):
138 """
139 Initialise the database
140 """
141 # Connect to database
142 db = location.database.Connection(
143 host=ns.database_host, database=ns.database_name,
144 user=ns.database_username, password=ns.database_password,
145 )
146
147 with db.transaction():
148 db.execute("""
149 -- announcements
150 CREATE TABLE IF NOT EXISTS announcements(network inet, autnum bigint,
151 first_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP,
152 last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
153 CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
154 CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
155 CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops);
156
157 -- autnums
158 CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
159 CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number);
160
161 -- countries
162 CREATE TABLE IF NOT EXISTS countries(
163 country_code text NOT NULL, name text NOT NULL, continent_code text NOT NULL);
164 CREATE UNIQUE INDEX IF NOT EXISTS countries_country_code ON countries(country_code);
165
166 -- networks
167 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
168 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
169 CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
170 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
171
172 -- overrides
173 CREATE TABLE IF NOT EXISTS autnum_overrides(
174 number bigint NOT NULL,
175 name text,
176 country text,
177 is_anonymous_proxy boolean,
178 is_satellite_provider boolean,
179 is_anycast boolean
180 );
181 CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number
182 ON autnum_overrides(number);
183
184 CREATE TABLE IF NOT EXISTS network_overrides(
185 network inet NOT NULL,
186 country text,
187 is_anonymous_proxy boolean,
188 is_satellite_provider boolean,
189 is_anycast boolean
190 );
191 CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
192 ON network_overrides(network);
193 CREATE INDEX IF NOT EXISTS network_overrides_search
194 ON network_overrides USING GIST(network inet_ops);
195 """)
196
197 return db
198
199 def handle_write(self, ns):
200 """
201 Compiles a database in libloc format out of what is in the database
202 """
203 # Allocate a writer
204 writer = location.Writer(ns.signing_key, ns.backup_signing_key)
205
206 # Set all metadata
207 if ns.vendor:
208 writer.vendor = ns.vendor
209
210 if ns.description:
211 writer.description = ns.description
212
213 if ns.license:
214 writer.license = ns.license
215
216 # Add all Autonomous Systems
217 log.info("Writing Autonomous Systems...")
218
219 # Select all ASes with a name
220 rows = self.db.query("""
221 SELECT
222 autnums.number AS number,
223 COALESCE(
224 (SELECT overrides.name FROM autnum_overrides overrides
225 WHERE overrides.number = autnums.number),
226 autnums.name
227 ) AS name
228 FROM autnums
229 WHERE name <> %s ORDER BY number
230 """, "")
231
232 for row in rows:
233 a = writer.add_as(row.number)
234 a.name = row.name
235
236 # Add all networks
237 log.info("Writing networks...")
238
239 # Select all known networks
240 rows = self.db.query("""
241 -- Return a list of those networks enriched with all
242 -- other information that we store in the database
243 SELECT
244 DISTINCT ON (network)
245 network,
246 autnum,
247
248 -- Country
249 COALESCE(
250 (
251 SELECT country FROM network_overrides overrides
252 WHERE networks.network <<= overrides.network
253 ORDER BY masklen(overrides.network) DESC
254 LIMIT 1
255 ),
256 (
257 SELECT country FROM autnum_overrides overrides
258 WHERE networks.autnum = overrides.number
259 ),
260 networks.country
261 ) AS country,
262
263 -- Flags
264 COALESCE(
265 (
266 SELECT is_anonymous_proxy FROM network_overrides overrides
267 WHERE networks.network <<= overrides.network
268 ORDER BY masklen(overrides.network) DESC
269 LIMIT 1
270 ),
271 (
272 SELECT is_anonymous_proxy FROM autnum_overrides overrides
273 WHERE networks.autnum = overrides.number
274 ),
275 FALSE
276 ) AS is_anonymous_proxy,
277 COALESCE(
278 (
279 SELECT is_satellite_provider FROM network_overrides overrides
280 WHERE networks.network <<= overrides.network
281 ORDER BY masklen(overrides.network) DESC
282 LIMIT 1
283 ),
284 (
285 SELECT is_satellite_provider FROM autnum_overrides overrides
286 WHERE networks.autnum = overrides.number
287 ),
288 FALSE
289 ) AS is_satellite_provider,
290 COALESCE(
291 (
292 SELECT is_anycast FROM network_overrides overrides
293 WHERE networks.network <<= overrides.network
294 ORDER BY masklen(overrides.network) DESC
295 LIMIT 1
296 ),
297 (
298 SELECT is_anycast FROM autnum_overrides overrides
299 WHERE networks.autnum = overrides.number
300 ),
301 FALSE
302 ) AS is_anycast
303 FROM (
304 SELECT
305 known_networks.network AS network,
306 announcements.autnum AS autnum,
307 networks.country AS country,
308
309 -- Must be part of returned values for ORDER BY clause
310 masklen(announcements.network) AS sort_a,
311 masklen(networks.network) AS sort_b
312 FROM (
313 SELECT network FROM announcements
314 UNION ALL
315 SELECT network FROM networks
316 UNION ALL
317 SELECT network FROM network_overrides
318 ) known_networks
319 LEFT JOIN
320 announcements ON known_networks.network <<= announcements.network
321 LEFT JOIN
322 networks ON known_networks.network <<= networks.network
323 ORDER BY
324 known_networks.network,
325 sort_a DESC,
326 sort_b DESC
327 ) networks
328 """)
329
330 for row in rows:
331 network = writer.add_network(row.network)
332
333 # Save country
334 if row.country:
335 network.country_code = row.country
336
337 # Save ASN
338 if row.autnum:
339 network.asn = row.autnum
340
341 # Set flags
342 if row.is_anonymous_proxy:
343 network.set_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY)
344
345 if row.is_satellite_provider:
346 network.set_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER)
347
348 if row.is_anycast:
349 network.set_flag(location.NETWORK_FLAG_ANYCAST)
350
351 # Add all countries
352 log.info("Writing countries...")
353 rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
354
355 for row in rows:
356 c = writer.add_country(row.country_code)
357 c.continent_code = row.continent_code
358 c.name = row.name
359
360 # Write everything to file
361 log.info("Writing database to file...")
362 for file in ns.file:
363 writer.write(file)
364
365 def handle_update_whois(self, ns):
366 downloader = location.importer.Downloader()
367
368 # Download all sources
369 with self.db.transaction():
370 # Create some temporary tables to store parsed data
371 self.db.execute("""
372 CREATE TEMPORARY TABLE _autnums(number integer, organization text)
373 ON COMMIT DROP;
374 CREATE UNIQUE INDEX _autnums_number ON _autnums(number);
375
376 CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
377 ON COMMIT DROP;
378 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
379
380 CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
381 ON COMMIT DROP;
382 CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
383 CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
384 """)
385
386 # Remove all previously imported content
387 self.db.execute("""
388 TRUNCATE TABLE networks;
389 """)
390
391 for source in location.importer.WHOIS_SOURCES:
392 with downloader.request(source, return_blocks=True) as f:
393 for block in f:
394 self._parse_block(block)
395
396 # Process all parsed networks from every RIR we happen to have access to,
397 # insert the largest network chunks into the networks table immediately...
398 families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
399
400 for family in (row.family for row in families):
401 smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
402
403 self.db.execute("INSERT INTO networks(network, country) \
404 SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
405
406 # ... determine any other prefixes for this network family, ...
407 prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
408 WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
409
410 # ... and insert networks with this prefix in case they provide additional
411 # information (i. e. subnet of a larger chunk with a different country)
412 for prefix in (row.prefix for row in prefixes):
413 self.db.execute("""
414 WITH candidates AS (
415 SELECT
416 _rirdata.network,
417 _rirdata.country
418 FROM
419 _rirdata
420 WHERE
421 family(_rirdata.network) = %s
422 AND
423 masklen(_rirdata.network) = %s
424 ),
425 filtered AS (
426 SELECT
427 DISTINCT ON (c.network)
428 c.network,
429 c.country,
430 masklen(networks.network),
431 networks.country AS parent_country
432 FROM
433 candidates c
434 LEFT JOIN
435 networks
436 ON
437 c.network << networks.network
438 ORDER BY
439 c.network,
440 masklen(networks.network) DESC NULLS LAST
441 )
442 INSERT INTO
443 networks(network, country)
444 SELECT
445 network,
446 country
447 FROM
448 filtered
449 WHERE
450 parent_country IS NULL
451 OR
452 country <> parent_country
453 ON CONFLICT DO NOTHING""",
454 family, prefix,
455 )
456
457 self.db.execute("""
458 INSERT INTO autnums(number, name)
459 SELECT _autnums.number, _organizations.name FROM _autnums
460 JOIN _organizations ON _autnums.organization = _organizations.handle
461 ON CONFLICT (number) DO UPDATE SET name = excluded.name;
462 """)
463
464 # Download all extended sources
465 for source in location.importer.EXTENDED_SOURCES:
466 with self.db.transaction():
467 # Download data
468 with downloader.request(source) as f:
469 for line in f:
470 self._parse_line(line)
471
472 def _check_parsed_network(self, network):
473 """
474 Assistive function to detect and subsequently sort out parsed
475 networks from RIR data (both Whois and so-called "extended sources"),
476 which are or have...
477
478 (a) not globally routable (RFC 1918 space, et al.)
479 (b) covering a too large chunk of the IP address space (prefix length
480 is < 7 for IPv4 networks, and < 10 for IPv6)
481 (c) "0.0.0.0" or "::" as a network address
482 (d) are too small for being publicly announced (we have decided not to
483 process them at the moment, as they significantly enlarge our
484 database without providing very helpful additional information)
485
486 This unfortunately is necessary due to brain-dead clutter across
487 various RIR databases, causing mismatches and eventually disruptions.
488
489 We will return False in case a network is not suitable for adding
490 it to our database, and True otherwise.
491 """
492
493 if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)):
494 return False
495
496 if not network.is_global:
497 log.warning("Skipping non-globally routable network: %s" % network)
498 return False
499
500 if network.version == 4:
501 if network.prefixlen < 7:
502 log.warning("Skipping too big IP chunk: %s" % network)
503 return False
504
505 if network.prefixlen > 24:
506 log.debug("Skipping network too small to be publicly announced: %s" % network)
507 return False
508
509 if str(network.network_address) == "0.0.0.0":
510 log.warning("Skipping network based on 0.0.0.0: %s" % network)
511 return False
512
513 elif network.version == 6:
514 if network.prefixlen < 10:
515 log.warning("Skipping too big IP chunk: %s" % network)
516 return False
517
518 if network.prefixlen > 48:
519 log.debug("Skipping network too small to be publicly announced: %s" % network)
520 return False
521
522 if str(network.network_address) == "::":
523 log.warning("Skipping network based on '::': %s" % network)
524 return False
525
526 else:
527 # This should not happen...
528 log.warning("Skipping network of unknown family, this should not happen: %s" % network)
529 return False
530
531 # In case we have made it here, the network is considered to
532 # be suitable for libloc consumption...
533 return True
534
535 def _parse_block(self, block):
536 # Get first line to find out what type of block this is
537 line = block[0]
538
539 # aut-num
540 if line.startswith("aut-num:"):
541 return self._parse_autnum_block(block)
542
543 # inetnum
544 if line.startswith("inet6num:") or line.startswith("inetnum:"):
545 return self._parse_inetnum_block(block)
546
547 # organisation
548 elif line.startswith("organisation:"):
549 return self._parse_org_block(block)
550
551 def _parse_autnum_block(self, block):
552 autnum = {}
553 for line in block:
554 # Split line
555 key, val = split_line(line)
556
557 if key == "aut-num":
558 m = re.match(r"^(AS|as)(\d+)", val)
559 if m:
560 autnum["asn"] = m.group(2)
561
562 elif key == "org":
563 autnum[key] = val.upper()
564
565 # Skip empty objects
566 if not autnum:
567 return
568
569 # Insert into database
570 self.db.execute("INSERT INTO _autnums(number, organization) \
571 VALUES(%s, %s) ON CONFLICT (number) DO UPDATE SET \
572 organization = excluded.organization",
573 autnum.get("asn"), autnum.get("org"),
574 )
575
576 def _parse_inetnum_block(self, block):
577 log.debug("Parsing inetnum block:")
578
579 inetnum = {}
580 for line in block:
581 log.debug(line)
582
583 # Split line
584 key, val = split_line(line)
585
586 # Filter any inetnum records which are only referring to IP space
587 # not managed by that specific RIR...
588 if key == "netname":
589 if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()):
590 log.warning("Skipping record indicating historic/orphaned data: %s" % val.strip())
591 return
592
593 if key == "inetnum":
594 start_address, delim, end_address = val.partition("-")
595
596 # Strip any excess space
597 start_address, end_address = start_address.rstrip(), end_address.strip()
598
599 # Convert to IP address
600 try:
601 start_address = ipaddress.ip_address(start_address)
602 end_address = ipaddress.ip_address(end_address)
603 except ValueError:
604 log.warning("Could not parse line: %s" % line)
605 return
606
607 # Set prefix to default
608 prefix = 32
609
610 # Count number of addresses in this subnet
611 num_addresses = int(end_address) - int(start_address)
612 if num_addresses:
613 prefix -= math.log(num_addresses, 2)
614
615 inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
616
617 elif key == "inet6num":
618 inetnum[key] = val
619
620 elif key == "country":
621 inetnum[key] = val.upper()
622
623 # Skip empty objects
624 if not inetnum or not "country" in inetnum:
625 return
626
627 # Skip objects with bogus country code 'ZZ'
628 if inetnum.get("country") == "ZZ":
629 log.warning("Skipping network with bogus country 'ZZ': %s" % \
630 (inetnum.get("inet6num") or inetnum.get("inetnum")))
631 return
632
633 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
634
635 if not self._check_parsed_network(network):
636 return
637
638 self.db.execute("INSERT INTO _rirdata(network, country) \
639 VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
640 "%s" % network, inetnum.get("country"),
641 )
642
643 def _parse_org_block(self, block):
644 org = {}
645 for line in block:
646 # Split line
647 key, val = split_line(line)
648
649 if key == "organisation":
650 org[key] = val.upper()
651 elif key == "org-name":
652 org[key] = val
653
654 # Skip empty objects
655 if not org:
656 return
657
658 self.db.execute("INSERT INTO _organizations(handle, name) \
659 VALUES(%s, %s) ON CONFLICT (handle) DO \
660 UPDATE SET name = excluded.name",
661 org.get("organisation"), org.get("org-name"),
662 )
663
664 def _parse_line(self, line):
665 # Skip version line
666 if line.startswith("2"):
667 return
668
669 # Skip comments
670 if line.startswith("#"):
671 return
672
673 try:
674 registry, country_code, type, line = line.split("|", 3)
675 except:
676 log.warning("Could not parse line: %s" % line)
677 return
678
679 # Skip any lines that are for stats only
680 if country_code == "*":
681 return
682
683 if type in ("ipv6", "ipv4"):
684 return self._parse_ip_line(country_code, type, line)
685
686 def _parse_ip_line(self, country, type, line):
687 try:
688 address, prefix, date, status, organization = line.split("|")
689 except ValueError:
690 organization = None
691
692 # Try parsing the line without organization
693 try:
694 address, prefix, date, status = line.split("|")
695 except ValueError:
696 log.warning("Unhandled line format: %s" % line)
697 return
698
699 # Skip anything that isn't properly assigned
700 if not status in ("assigned", "allocated"):
701 return
702
703 # Cast prefix into an integer
704 try:
705 prefix = int(prefix)
706 except:
707 log.warning("Invalid prefix: %s" % prefix)
708 return
709
710 # Fix prefix length for IPv4
711 if type == "ipv4":
712 prefix = 32 - int(math.log(prefix, 2))
713
714 # Try to parse the address
715 try:
716 network = ipaddress.ip_network("%s/%s" % (address, prefix), strict=False)
717 except ValueError:
718 log.warning("Invalid IP address: %s" % address)
719 return
720
721 if not self._check_parsed_network(network):
722 return
723
724 self.db.execute("INSERT INTO networks(network, country) \
725 VALUES(%s, %s) ON CONFLICT (network) DO \
726 UPDATE SET country = excluded.country",
727 "%s" % network, country,
728 )
729
730 def handle_update_announcements(self, ns):
731 server = ns.server[0]
732
733 with self.db.transaction():
734 if server.startswith("/"):
735 self._handle_update_announcements_from_bird(server)
736 else:
737 self._handle_update_announcements_from_telnet(server)
738
739 # Purge anything we never want here
740 self.db.execute("""
741 -- Delete default routes
742 DELETE FROM announcements WHERE network = '::/0' OR network = '0.0.0.0/0';
743
744 -- Delete anything that is not global unicast address space
745 DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3';
746
747 -- DELETE "current network" address space
748 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '0.0.0.0/8';
749
750 -- DELETE local loopback address space
751 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '127.0.0.0/8';
752
753 -- DELETE RFC 1918 address space
754 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8';
755 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12';
756 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16';
757
758 -- DELETE test, benchmark and documentation address space
759 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.0.0/24';
760 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.2.0/24';
761 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.18.0.0/15';
762 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.51.100.0/24';
763 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '203.0.113.0/24';
764
765 -- DELETE CGNAT address space (RFC 6598)
766 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '100.64.0.0/10';
767
768 -- DELETE link local address space
769 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '169.254.0.0/16';
770
771 -- DELETE IPv6 to IPv4 (6to4) address space (RFC 3068)
772 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.88.99.0/24';
773 DELETE FROM announcements WHERE family(network) = 6 AND network <<= '2002::/16';
774
775 -- DELETE multicast and reserved address space
776 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '224.0.0.0/4';
777 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '240.0.0.0/4';
778
779 -- Delete networks that are too small to be in the global routing table
780 DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48;
781 DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24;
782
783 -- Delete any non-public or reserved ASNs
784 DELETE FROM announcements WHERE NOT (
785 (autnum >= 1 AND autnum <= 23455)
786 OR
787 (autnum >= 23457 AND autnum <= 64495)
788 OR
789 (autnum >= 131072 AND autnum <= 4199999999)
790 );
791
792 -- Delete everything that we have not seen for 14 days
793 DELETE FROM announcements WHERE last_seen_at <= CURRENT_TIMESTAMP - INTERVAL '14 days';
794 """)
795
796 def _handle_update_announcements_from_bird(self, server):
797 # Pre-compile the regular expression for faster searching
798 route = re.compile(b"^\s(.+?)\s+.+?\[AS(.*?).\]$")
799
800 log.info("Requesting routing table from Bird (%s)" % server)
801
802 # Send command to list all routes
803 for line in self._bird_cmd(server, "show route"):
804 m = route.match(line)
805 if not m:
806 log.debug("Could not parse line: %s" % line.decode())
807 continue
808
809 # Fetch the extracted network and ASN
810 network, autnum = m.groups()
811
812 # Insert it into the database
813 self.db.execute("INSERT INTO announcements(network, autnum) \
814 VALUES(%s, %s) ON CONFLICT (network) DO \
815 UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
816 network.decode(), autnum.decode(),
817 )
818
819 def _handle_update_announcements_from_telnet(self, server):
820 # Pre-compile regular expression for routes
821 route = re.compile(b"^\*[\s\>]i([^\s]+).+?(\d+)\si\r\n", re.MULTILINE|re.DOTALL)
822
823 with telnetlib.Telnet(server) as t:
824 # Enable debug mode
825 #if ns.debug:
826 # t.set_debuglevel(10)
827
828 # Wait for console greeting
829 greeting = t.read_until(b"> ", timeout=30)
830 if not greeting:
831 log.error("Could not get a console prompt")
832 return 1
833
834 # Disable pagination
835 t.write(b"terminal length 0\n")
836
837 # Wait for the prompt to return
838 t.read_until(b"> ")
839
840 # Fetch the routing tables
841 for protocol in ("ipv6", "ipv4"):
842 log.info("Requesting %s routing table" % protocol)
843
844 # Request the full unicast routing table
845 t.write(b"show bgp %s unicast\n" % protocol.encode())
846
847 # Read entire header which ends with "Path"
848 t.read_until(b"Path\r\n")
849
850 while True:
851 # Try reading a full entry
852 # Those might be broken across multiple lines but ends with i
853 line = t.read_until(b"i\r\n", timeout=5)
854 if not line:
855 break
856
857 # Show line for debugging
858 #log.debug(repr(line))
859
860 # Try finding a route in here
861 m = route.match(line)
862 if m:
863 network, autnum = m.groups()
864
865 # Convert network to string
866 network = network.decode()
867
868 # Append /24 for IPv4 addresses
869 if not "/" in network and not ":" in network:
870 network = "%s/24" % network
871
872 # Convert AS number to integer
873 autnum = int(autnum)
874
875 log.info("Found announcement for %s by %s" % (network, autnum))
876
877 self.db.execute("INSERT INTO announcements(network, autnum) \
878 VALUES(%s, %s) ON CONFLICT (network) DO \
879 UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
880 network, autnum,
881 )
882
883 log.info("Finished reading the %s routing table" % protocol)
884
885 def _bird_cmd(self, socket_path, command):
886 # Connect to the socket
887 s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
888 s.connect(socket_path)
889
890 # Allocate some buffer
891 buffer = b""
892
893 # Send the command
894 s.send(b"%s\n" % command.encode())
895
896 while True:
897 # Fill up the buffer
898 buffer += s.recv(4096)
899
900 while True:
901 # Search for the next newline
902 pos = buffer.find(b"\n")
903
904 # If we cannot find one, we go back and read more data
905 if pos <= 0:
906 break
907
908 # Cut after the newline character
909 pos += 1
910
911 # Split the line we want and keep the rest in buffer
912 line, buffer = buffer[:pos], buffer[pos:]
913
914 # Look for the end-of-output indicator
915 if line == b"0000 \n":
916 return
917
918 # Otherwise return the line
919 yield line
920
921 def handle_update_overrides(self, ns):
922 with self.db.transaction():
923 # Drop all data that we have
924 self.db.execute("""
925 TRUNCATE TABLE autnum_overrides;
926 TRUNCATE TABLE network_overrides;
927 """)
928
929 for file in ns.files:
930 log.info("Reading %s..." % file)
931
932 with open(file, "rb") as f:
933 for type, block in location.importer.read_blocks(f):
934 if type == "net":
935 network = block.get("net")
936 # Try to parse and normalise the network
937 try:
938 network = ipaddress.ip_network(network, strict=False)
939 except ValueError as e:
940 log.warning("Invalid IP network: %s: %s" % (network, e))
941 continue
942
943 # Prevent that we overwrite all networks
944 if network.prefixlen == 0:
945 log.warning("Skipping %s: You cannot overwrite default" % network)
946 continue
947
948 self.db.execute("""
949 INSERT INTO network_overrides(
950 network,
951 country,
952 is_anonymous_proxy,
953 is_satellite_provider,
954 is_anycast
955 ) VALUES (%s, %s, %s, %s, %s)
956 ON CONFLICT (network) DO NOTHING""",
957 "%s" % network,
958 block.get("country"),
959 self._parse_bool(block, "is-anonymous-proxy"),
960 self._parse_bool(block, "is-satellite-provider"),
961 self._parse_bool(block, "is-anycast"),
962 )
963
964 elif type == "aut-num":
965 autnum = block.get("aut-num")
966
967 # Check if AS number begins with "AS"
968 if not autnum.startswith("AS"):
969 log.warning("Invalid AS number: %s" % autnum)
970 continue
971
972 # Strip "AS"
973 autnum = autnum[2:]
974
975 self.db.execute("""
976 INSERT INTO autnum_overrides(
977 number,
978 name,
979 country,
980 is_anonymous_proxy,
981 is_satellite_provider,
982 is_anycast
983 ) VALUES(%s, %s, %s, %s, %s, %s)
984 ON CONFLICT DO NOTHING""",
985 autnum,
986 block.get("name"),
987 block.get("country"),
988 self._parse_bool(block, "is-anonymous-proxy"),
989 self._parse_bool(block, "is-satellite-provider"),
990 self._parse_bool(block, "is-anycast"),
991 )
992
993 else:
994 log.warning("Unsupported type: %s" % type)
995
996 @staticmethod
997 def _parse_bool(block, key):
998 val = block.get(key)
999
1000 # There is no point to proceed when we got None
1001 if val is None:
1002 return
1003
1004 # Convert to lowercase
1005 val = val.lower()
1006
1007 # True
1008 if val in ("yes", "1"):
1009 return True
1010
1011 # False
1012 if val in ("no", "0"):
1013 return False
1014
1015 # Default to None
1016 return None
1017
1018 def handle_import_countries(self, ns):
1019 with self.db.transaction():
1020 # Drop all data that we have
1021 self.db.execute("TRUNCATE TABLE countries")
1022
1023 for file in ns.file:
1024 for line in file:
1025 line = line.rstrip()
1026
1027 # Ignore any comments
1028 if line.startswith("#"):
1029 continue
1030
1031 try:
1032 country_code, continent_code, name = line.split(maxsplit=2)
1033 except:
1034 log.warning("Could not parse line: %s" % line)
1035 continue
1036
1037 self.db.execute("INSERT INTO countries(country_code, name, continent_code) \
1038 VALUES(%s, %s, %s) ON CONFLICT DO NOTHING", country_code, name, continent_code)
1039
1040
1041 def split_line(line):
1042 key, colon, val = line.partition(":")
1043
1044 # Strip any excess space
1045 key = key.strip()
1046 val = val.strip()
1047
1048 return key, val
1049
1050 def main():
1051 # Run the command line interface
1052 c = CLI()
1053 c.run()
1054
1055 main()
A library to determine the location of someone on the Internet