]> git.ipfire.org Git - people/ms/libloc.git/blob - src/python/location-importer.in
projects / people / ms / libloc.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
importer: Drop EDROP as it has been merged into DROP
[people/ms/libloc.git] / src / python / location-importer.in
1 #!/usr/bin/python3
2 ###############################################################################
3 # #
4 # libloc - A library to determine the location of someone on the Internet #
5 # #
6 # Copyright (C) 2020-2021 IPFire Development Team <info@ipfire.org> #
7 # #
8 # This library is free software; you can redistribute it and/or #
9 # modify it under the terms of the GNU Lesser General Public #
10 # License as published by the Free Software Foundation; either #
11 # version 2.1 of the License, or (at your option) any later version. #
12 # #
13 # This library is distributed in the hope that it will be useful, #
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
16 # Lesser General Public License for more details. #
17 # #
18 ###############################################################################
19
20 import argparse
21 import ipaddress
22 import logging
23 import math
24 import re
25 import socket
26 import sys
27 import telnetlib
28
29 # Load our location module
30 import location
31 import location.database
32 import location.importer
33 from location.i18n import _
34
35 # Initialise logging
36 log = logging.getLogger("location.importer")
37 log.propagate = 1
38
39 class CLI(object):
40 def parse_cli(self):
41 parser = argparse.ArgumentParser(
42 description=_("Location Importer Command Line Interface"),
43 )
44 subparsers = parser.add_subparsers()
45
46 # Global configuration flags
47 parser.add_argument("--debug", action="store_true",
48 help=_("Enable debug output"))
49 parser.add_argument("--quiet", action="store_true",
50 help=_("Enable quiet mode"))
51
52 # version
53 parser.add_argument("--version", action="version",
54 version="%(prog)s @VERSION@")
55
56 # Database
57 parser.add_argument("--database-host", required=True,
58 help=_("Database Hostname"), metavar=_("HOST"))
59 parser.add_argument("--database-name", required=True,
60 help=_("Database Name"), metavar=_("NAME"))
61 parser.add_argument("--database-username", required=True,
62 help=_("Database Username"), metavar=_("USERNAME"))
63 parser.add_argument("--database-password", required=True,
64 help=_("Database Password"), metavar=_("PASSWORD"))
65
66 # Write Database
67 write = subparsers.add_parser("write", help=_("Write database to file"))
68 write.set_defaults(func=self.handle_write)
69 write.add_argument("file", nargs=1, help=_("Database File"))
70 write.add_argument("--signing-key", nargs="?", type=open, help=_("Signing Key"))
71 write.add_argument("--backup-signing-key", nargs="?", type=open, help=_("Backup Signing Key"))
72 write.add_argument("--vendor", nargs="?", help=_("Sets the vendor"))
73 write.add_argument("--description", nargs="?", help=_("Sets a description"))
74 write.add_argument("--license", nargs="?", help=_("Sets the license"))
75 write.add_argument("--version", type=int, help=_("Database Format Version"))
76
77 # Update WHOIS
78 update_whois = subparsers.add_parser("update-whois", help=_("Update WHOIS Information"))
79 update_whois.set_defaults(func=self.handle_update_whois)
80
81 # Update announcements
82 update_announcements = subparsers.add_parser("update-announcements",
83 help=_("Update BGP Annoucements"))
84 update_announcements.set_defaults(func=self.handle_update_announcements)
85 update_announcements.add_argument("server", nargs=1,
86 help=_("Route Server to connect to"), metavar=_("SERVER"))
87
88 # Update overrides
89 update_overrides = subparsers.add_parser("update-overrides",
90 help=_("Update overrides"),
91 )
92 update_overrides.add_argument(
93 "files", nargs="+", help=_("Files to import"),
94 )
95 update_overrides.set_defaults(func=self.handle_update_overrides)
96
97 # Import countries
98 import_countries = subparsers.add_parser("import-countries",
99 help=_("Import countries"),
100 )
101 import_countries.add_argument("file", nargs=1, type=argparse.FileType("r"),
102 help=_("File to import"))
103 import_countries.set_defaults(func=self.handle_import_countries)
104
105 args = parser.parse_args()
106
107 # Configure logging
108 if args.debug:
109 location.logger.set_level(logging.DEBUG)
110 elif args.quiet:
111 location.logger.set_level(logging.WARNING)
112
113 # Print usage if no action was given
114 if not "func" in args:
115 parser.print_usage()
116 sys.exit(2)
117
118 return args
119
120 def run(self):
121 # Parse command line arguments
122 args = self.parse_cli()
123
124 # Initialise database
125 self.db = self._setup_database(args)
126
127 # Call function
128 ret = args.func(args)
129
130 # Return with exit code
131 if ret:
132 sys.exit(ret)
133
134 # Otherwise just exit
135 sys.exit(0)
136
137 def _setup_database(self, ns):
138 """
139 Initialise the database
140 """
141 # Connect to database
142 db = location.database.Connection(
143 host=ns.database_host, database=ns.database_name,
144 user=ns.database_username, password=ns.database_password,
145 )
146
147 with db.transaction():
148 db.execute("""
149 -- announcements
150 CREATE TABLE IF NOT EXISTS announcements(network inet, autnum bigint,
151 first_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP,
152 last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
153 CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
154 CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
155 CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops);
156
157 -- autnums
158 CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
159 CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number);
160
161 -- countries
162 CREATE TABLE IF NOT EXISTS countries(
163 country_code text NOT NULL, name text NOT NULL, continent_code text NOT NULL);
164 CREATE UNIQUE INDEX IF NOT EXISTS countries_country_code ON countries(country_code);
165
166 -- networks
167 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
168 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
169 CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
170 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
171
172 -- overrides
173 CREATE TABLE IF NOT EXISTS autnum_overrides(
174 number bigint NOT NULL,
175 name text,
176 country text,
177 is_anonymous_proxy boolean,
178 is_satellite_provider boolean,
179 is_anycast boolean
180 );
181 CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number
182 ON autnum_overrides(number);
183
184 CREATE TABLE IF NOT EXISTS network_overrides(
185 network inet NOT NULL,
186 country text,
187 is_anonymous_proxy boolean,
188 is_satellite_provider boolean,
189 is_anycast boolean
190 );
191 CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
192 ON network_overrides(network);
193 CREATE INDEX IF NOT EXISTS network_overrides_search
194 ON network_overrides USING GIST(network inet_ops);
195 """)
196
197 return db
198
199 def handle_write(self, ns):
200 """
201 Compiles a database in libloc format out of what is in the database
202 """
203 # Allocate a writer
204 writer = location.Writer(ns.signing_key, ns.backup_signing_key)
205
206 # Set all metadata
207 if ns.vendor:
208 writer.vendor = ns.vendor
209
210 if ns.description:
211 writer.description = ns.description
212
213 if ns.license:
214 writer.license = ns.license
215
216 # Add all Autonomous Systems
217 log.info("Writing Autonomous Systems...")
218
219 # Select all ASes with a name
220 rows = self.db.query("""
221 SELECT
222 autnums.number AS number,
223 COALESCE(
224 (SELECT overrides.name FROM autnum_overrides overrides
225 WHERE overrides.number = autnums.number),
226 autnums.name
227 ) AS name
228 FROM autnums
229 WHERE name <> %s ORDER BY number
230 """, "")
231
232 for row in rows:
233 a = writer.add_as(row.number)
234 a.name = row.name
235
236 # Add all networks
237 log.info("Writing networks...")
238
239 # Select all known networks
240 rows = self.db.query("""
241 -- Return a list of those networks enriched with all
242 -- other information that we store in the database
243 SELECT
244 DISTINCT ON (network)
245 network,
246 autnum,
247
248 -- Country
249 COALESCE(
250 (
251 SELECT country FROM network_overrides overrides
252 WHERE networks.network <<= overrides.network
253 ORDER BY masklen(overrides.network) DESC
254 LIMIT 1
255 ),
256 (
257 SELECT country FROM autnum_overrides overrides
258 WHERE networks.autnum = overrides.number
259 ),
260 networks.country
261 ) AS country,
262
263 -- Flags
264 COALESCE(
265 (
266 SELECT is_anonymous_proxy FROM network_overrides overrides
267 WHERE networks.network <<= overrides.network
268 ORDER BY masklen(overrides.network) DESC
269 LIMIT 1
270 ),
271 (
272 SELECT is_anonymous_proxy FROM autnum_overrides overrides
273 WHERE networks.autnum = overrides.number
274 ),
275 FALSE
276 ) AS is_anonymous_proxy,
277 COALESCE(
278 (
279 SELECT is_satellite_provider FROM network_overrides overrides
280 WHERE networks.network <<= overrides.network
281 ORDER BY masklen(overrides.network) DESC
282 LIMIT 1
283 ),
284 (
285 SELECT is_satellite_provider FROM autnum_overrides overrides
286 WHERE networks.autnum = overrides.number
287 ),
288 FALSE
289 ) AS is_satellite_provider,
290 COALESCE(
291 (
292 SELECT is_anycast FROM network_overrides overrides
293 WHERE networks.network <<= overrides.network
294 ORDER BY masklen(overrides.network) DESC
295 LIMIT 1
296 ),
297 (
298 SELECT is_anycast FROM autnum_overrides overrides
299 WHERE networks.autnum = overrides.number
300 ),
301 FALSE
302 ) AS is_anycast
303 FROM (
304 SELECT
305 known_networks.network AS network,
306 announcements.autnum AS autnum,
307 networks.country AS country,
308
309 -- Must be part of returned values for ORDER BY clause
310 masklen(announcements.network) AS sort_a,
311 masklen(networks.network) AS sort_b
312 FROM (
313 SELECT network FROM announcements
314 UNION ALL
315 SELECT network FROM networks
316 UNION ALL
317 SELECT network FROM network_overrides
318 ) known_networks
319 LEFT JOIN
320 announcements ON known_networks.network <<= announcements.network
321 LEFT JOIN
322 networks ON known_networks.network <<= networks.network
323 ORDER BY
324 known_networks.network,
325 sort_a DESC,
326 sort_b DESC
327 ) networks
328 """)
329
330 for row in rows:
331 network = writer.add_network(row.network)
332
333 # Save country
334 if row.country:
335 network.country_code = row.country
336
337 # Save ASN
338 if row.autnum:
339 network.asn = row.autnum
340
341 # Set flags
342 if row.is_anonymous_proxy:
343 network.set_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY)
344
345 if row.is_satellite_provider:
346 network.set_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER)
347
348 if row.is_anycast:
349 network.set_flag(location.NETWORK_FLAG_ANYCAST)
350
351 # Add all countries
352 log.info("Writing countries...")
353 rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
354
355 for row in rows:
356 c = writer.add_country(row.country_code)
357 c.continent_code = row.continent_code
358 c.name = row.name
359
360 # Write everything to file
361 log.info("Writing database to file...")
362 for file in ns.file:
363 writer.write(file)
364
365 def handle_update_whois(self, ns):
366 downloader = location.importer.Downloader()
367
368 # Download all sources
369 with self.db.transaction():
370 # Create some temporary tables to store parsed data
371 self.db.execute("""
372 CREATE TEMPORARY TABLE _autnums(number integer, organization text)
373 ON COMMIT DROP;
374 CREATE UNIQUE INDEX _autnums_number ON _autnums(number);
375
376 CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
377 ON COMMIT DROP;
378 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
379
380 CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
381 ON COMMIT DROP;
382 CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
383 CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
384 """)
385
386 # Remove all previously imported content
387 self.db.execute("""
388 TRUNCATE TABLE networks;
389 """)
390
391 # Fetch all valid country codes to check parsed networks aganist...
392 rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
393 validcountries = []
394
395 for row in rows:
396 validcountries.append(row.country_code)
397
398 for source in location.importer.WHOIS_SOURCES:
399 with downloader.request(source, return_blocks=True) as f:
400 for block in f:
401 self._parse_block(block, validcountries)
402
403 # Process all parsed networks from every RIR we happen to have access to,
404 # insert the largest network chunks into the networks table immediately...
405 families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
406
407 for family in (row.family for row in families):
408 smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
409
410 self.db.execute("INSERT INTO networks(network, country) \
411 SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
412
413 # ... determine any other prefixes for this network family, ...
414 prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
415 WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
416
417 # ... and insert networks with this prefix in case they provide additional
418 # information (i. e. subnet of a larger chunk with a different country)
419 for prefix in (row.prefix for row in prefixes):
420 self.db.execute("""
421 WITH candidates AS (
422 SELECT
423 _rirdata.network,
424 _rirdata.country
425 FROM
426 _rirdata
427 WHERE
428 family(_rirdata.network) = %s
429 AND
430 masklen(_rirdata.network) = %s
431 ),
432 filtered AS (
433 SELECT
434 DISTINCT ON (c.network)
435 c.network,
436 c.country,
437 masklen(networks.network),
438 networks.country AS parent_country
439 FROM
440 candidates c
441 LEFT JOIN
442 networks
443 ON
444 c.network << networks.network
445 ORDER BY
446 c.network,
447 masklen(networks.network) DESC NULLS LAST
448 )
449 INSERT INTO
450 networks(network, country)
451 SELECT
452 network,
453 country
454 FROM
455 filtered
456 WHERE
457 parent_country IS NULL
458 OR
459 country <> parent_country
460 ON CONFLICT DO NOTHING""",
461 family, prefix,
462 )
463
464 self.db.execute("""
465 INSERT INTO autnums(number, name)
466 SELECT _autnums.number, _organizations.name FROM _autnums
467 JOIN _organizations ON _autnums.organization = _organizations.handle
468 ON CONFLICT (number) DO UPDATE SET name = excluded.name;
469 """)
470
471 # Download all extended sources
472 for source in location.importer.EXTENDED_SOURCES:
473 with self.db.transaction():
474 # Download data
475 with downloader.request(source) as f:
476 for line in f:
477 self._parse_line(line, validcountries)
478
479 def _check_parsed_network(self, network):
480 """
481 Assistive function to detect and subsequently sort out parsed
482 networks from RIR data (both Whois and so-called "extended sources"),
483 which are or have...
484
485 (a) not globally routable (RFC 1918 space, et al.)
486 (b) covering a too large chunk of the IP address space (prefix length
487 is < 7 for IPv4 networks, and < 10 for IPv6)
488 (c) "0.0.0.0" or "::" as a network address
489 (d) are too small for being publicly announced (we have decided not to
490 process them at the moment, as they significantly enlarge our
491 database without providing very helpful additional information)
492
493 This unfortunately is necessary due to brain-dead clutter across
494 various RIR databases, causing mismatches and eventually disruptions.
495
496 We will return False in case a network is not suitable for adding
497 it to our database, and True otherwise.
498 """
499
500 if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)):
501 return False
502
503 if not network.is_global:
504 log.debug("Skipping non-globally routable network: %s" % network)
505 return False
506
507 if network.version == 4:
508 if network.prefixlen < 7:
509 log.debug("Skipping too big IP chunk: %s" % network)
510 return False
511
512 if network.prefixlen > 24:
513 log.debug("Skipping network too small to be publicly announced: %s" % network)
514 return False
515
516 if str(network.network_address) == "0.0.0.0":
517 log.debug("Skipping network based on 0.0.0.0: %s" % network)
518 return False
519
520 elif network.version == 6:
521 if network.prefixlen < 10:
522 log.debug("Skipping too big IP chunk: %s" % network)
523 return False
524
525 if network.prefixlen > 48:
526 log.debug("Skipping network too small to be publicly announced: %s" % network)
527 return False
528
529 if str(network.network_address) == "::":
530 log.debug("Skipping network based on '::': %s" % network)
531 return False
532
533 else:
534 # This should not happen...
535 log.warning("Skipping network of unknown family, this should not happen: %s" % network)
536 return False
537
538 # In case we have made it here, the network is considered to
539 # be suitable for libloc consumption...
540 return True
541
542 def _parse_block(self, block, validcountries = None):
543 # Get first line to find out what type of block this is
544 line = block[0]
545
546 # aut-num
547 if line.startswith("aut-num:"):
548 return self._parse_autnum_block(block)
549
550 # inetnum
551 if line.startswith("inet6num:") or line.startswith("inetnum:"):
552 return self._parse_inetnum_block(block, validcountries)
553
554 # organisation
555 elif line.startswith("organisation:"):
556 return self._parse_org_block(block)
557
558 def _parse_autnum_block(self, block):
559 autnum = {}
560 for line in block:
561 # Split line
562 key, val = split_line(line)
563
564 if key == "aut-num":
565 m = re.match(r"^(AS|as)(\d+)", val)
566 if m:
567 autnum["asn"] = m.group(2)
568
569 elif key == "org":
570 autnum[key] = val.upper()
571
572 # Skip empty objects
573 if not autnum:
574 return
575
576 # Insert into database
577 self.db.execute("INSERT INTO _autnums(number, organization) \
578 VALUES(%s, %s) ON CONFLICT (number) DO UPDATE SET \
579 organization = excluded.organization",
580 autnum.get("asn"), autnum.get("org"),
581 )
582
583 def _parse_inetnum_block(self, block, validcountries = None):
584 log.debug("Parsing inetnum block:")
585
586 inetnum = {}
587 for line in block:
588 log.debug(line)
589
590 # Split line
591 key, val = split_line(line)
592
593 # Filter any inetnum records which are only referring to IP space
594 # not managed by that specific RIR...
595 if key == "netname":
596 if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()):
597 log.debug("Skipping record indicating historic/orphaned data: %s" % val.strip())
598 return
599
600 if key == "inetnum":
601 start_address, delim, end_address = val.partition("-")
602
603 # Strip any excess space
604 start_address, end_address = start_address.rstrip(), end_address.strip()
605
606 # Convert to IP address
607 try:
608 start_address = ipaddress.ip_address(start_address)
609 end_address = ipaddress.ip_address(end_address)
610 except ValueError:
611 log.warning("Could not parse line: %s" % line)
612 return
613
614 inetnum["inetnum"] = list(ipaddress.summarize_address_range(start_address, end_address))
615
616 elif key == "inet6num":
617 inetnum[key] = [ipaddress.ip_network(val, strict=False)]
618
619 elif key == "country":
620 inetnum[key] = val.upper()
621
622 # Skip empty objects
623 if not inetnum or not "country" in inetnum:
624 return
625
626 # Skip objects with unknown country codes
627 if validcountries and inetnum.get("country") not in validcountries:
628 log.warning("Skipping network with bogus country '%s': %s" % \
629 (inetnum.get("country"), inetnum.get("inet6num") or inetnum.get("inetnum")))
630 return
631
632 # Iterate through all networks enumerated from above, check them for plausibility and insert
633 # them into the database, if _check_parsed_network() succeeded
634 for single_network in inetnum.get("inet6num") or inetnum.get("inetnum"):
635 if self._check_parsed_network(single_network):
636 self.db.execute("INSERT INTO _rirdata(network, country) \
637 VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
638 "%s" % single_network, inetnum.get("country"),
639 )
640
641 def _parse_org_block(self, block):
642 org = {}
643 for line in block:
644 # Split line
645 key, val = split_line(line)
646
647 if key == "organisation":
648 org[key] = val.upper()
649 elif key == "org-name":
650 org[key] = val
651
652 # Skip empty objects
653 if not org:
654 return
655
656 self.db.execute("INSERT INTO _organizations(handle, name) \
657 VALUES(%s, %s) ON CONFLICT (handle) DO \
658 UPDATE SET name = excluded.name",
659 org.get("organisation"), org.get("org-name"),
660 )
661
662 def _parse_line(self, line, validcountries = None):
663 # Skip version line
664 if line.startswith("2"):
665 return
666
667 # Skip comments
668 if line.startswith("#"):
669 return
670
671 try:
672 registry, country_code, type, line = line.split("|", 3)
673 except:
674 log.warning("Could not parse line: %s" % line)
675 return
676
677 # Skip any lines that are for stats only or do not have a country
678 # code at all (avoids log spam below)
679 if not country_code or country_code == '*':
680 return
681
682 # Skip objects with unknown country codes
683 if validcountries and country_code not in validcountries:
684 log.warning("Skipping line with bogus country '%s': %s" % \
685 (country_code, line))
686 return
687
688 if type in ("ipv6", "ipv4"):
689 return self._parse_ip_line(country_code, type, line)
690
691 def _parse_ip_line(self, country, type, line):
692 try:
693 address, prefix, date, status, organization = line.split("|")
694 except ValueError:
695 organization = None
696
697 # Try parsing the line without organization
698 try:
699 address, prefix, date, status = line.split("|")
700 except ValueError:
701 log.warning("Unhandled line format: %s" % line)
702 return
703
704 # Skip anything that isn't properly assigned
705 if not status in ("assigned", "allocated"):
706 return
707
708 # Cast prefix into an integer
709 try:
710 prefix = int(prefix)
711 except:
712 log.warning("Invalid prefix: %s" % prefix)
713 return
714
715 # Fix prefix length for IPv4
716 if type == "ipv4":
717 prefix = 32 - int(math.log(prefix, 2))
718
719 # Try to parse the address
720 try:
721 network = ipaddress.ip_network("%s/%s" % (address, prefix), strict=False)
722 except ValueError:
723 log.warning("Invalid IP address: %s" % address)
724 return
725
726 if not self._check_parsed_network(network):
727 return
728
729 self.db.execute("INSERT INTO networks(network, country) \
730 VALUES(%s, %s) ON CONFLICT (network) DO \
731 UPDATE SET country = excluded.country",
732 "%s" % network, country,
733 )
734
735 def handle_update_announcements(self, ns):
736 server = ns.server[0]
737
738 with self.db.transaction():
739 if server.startswith("/"):
740 self._handle_update_announcements_from_bird(server)
741 else:
742 self._handle_update_announcements_from_telnet(server)
743
744 # Purge anything we never want here
745 self.db.execute("""
746 -- Delete default routes
747 DELETE FROM announcements WHERE network = '::/0' OR network = '0.0.0.0/0';
748
749 -- Delete anything that is not global unicast address space
750 DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3';
751
752 -- DELETE "current network" address space
753 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '0.0.0.0/8';
754
755 -- DELETE local loopback address space
756 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '127.0.0.0/8';
757
758 -- DELETE RFC 1918 address space
759 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8';
760 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12';
761 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16';
762
763 -- DELETE test, benchmark and documentation address space
764 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.0.0/24';
765 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.2.0/24';
766 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.18.0.0/15';
767 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.51.100.0/24';
768 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '203.0.113.0/24';
769
770 -- DELETE CGNAT address space (RFC 6598)
771 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '100.64.0.0/10';
772
773 -- DELETE link local address space
774 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '169.254.0.0/16';
775
776 -- DELETE IPv6 to IPv4 (6to4) address space (RFC 3068)
777 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.88.99.0/24';
778 DELETE FROM announcements WHERE family(network) = 6 AND network <<= '2002::/16';
779
780 -- DELETE multicast and reserved address space
781 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '224.0.0.0/4';
782 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '240.0.0.0/4';
783
784 -- Delete networks that are too small to be in the global routing table
785 DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48;
786 DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24;
787
788 -- Delete any non-public or reserved ASNs
789 DELETE FROM announcements WHERE NOT (
790 (autnum >= 1 AND autnum <= 23455)
791 OR
792 (autnum >= 23457 AND autnum <= 64495)
793 OR
794 (autnum >= 131072 AND autnum <= 4199999999)
795 );
796
797 -- Delete everything that we have not seen for 14 days
798 DELETE FROM announcements WHERE last_seen_at <= CURRENT_TIMESTAMP - INTERVAL '14 days';
799 """)
800
801 def _handle_update_announcements_from_bird(self, server):
802 # Pre-compile the regular expression for faster searching
803 route = re.compile(b"^\s(.+?)\s+.+?\[AS(.*?).\]$")
804
805 log.info("Requesting routing table from Bird (%s)" % server)
806
807 # Send command to list all routes
808 for line in self._bird_cmd(server, "show route"):
809 m = route.match(line)
810 if not m:
811 log.debug("Could not parse line: %s" % line.decode())
812 continue
813
814 # Fetch the extracted network and ASN
815 network, autnum = m.groups()
816
817 # Insert it into the database
818 self.db.execute("INSERT INTO announcements(network, autnum) \
819 VALUES(%s, %s) ON CONFLICT (network) DO \
820 UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
821 network.decode(), autnum.decode(),
822 )
823
824 def _handle_update_announcements_from_telnet(self, server):
825 # Pre-compile regular expression for routes
826 route = re.compile(b"^\*[\s\>]i([^\s]+).+?(\d+)\si\r\n", re.MULTILINE|re.DOTALL)
827
828 with telnetlib.Telnet(server) as t:
829 # Enable debug mode
830 #if ns.debug:
831 # t.set_debuglevel(10)
832
833 # Wait for console greeting
834 greeting = t.read_until(b"> ", timeout=30)
835 if not greeting:
836 log.error("Could not get a console prompt")
837 return 1
838
839 # Disable pagination
840 t.write(b"terminal length 0\n")
841
842 # Wait for the prompt to return
843 t.read_until(b"> ")
844
845 # Fetch the routing tables
846 for protocol in ("ipv6", "ipv4"):
847 log.info("Requesting %s routing table" % protocol)
848
849 # Request the full unicast routing table
850 t.write(b"show bgp %s unicast\n" % protocol.encode())
851
852 # Read entire header which ends with "Path"
853 t.read_until(b"Path\r\n")
854
855 while True:
856 # Try reading a full entry
857 # Those might be broken across multiple lines but ends with i
858 line = t.read_until(b"i\r\n", timeout=5)
859 if not line:
860 break
861
862 # Show line for debugging
863 #log.debug(repr(line))
864
865 # Try finding a route in here
866 m = route.match(line)
867 if m:
868 network, autnum = m.groups()
869
870 # Convert network to string
871 network = network.decode()
872
873 # Append /24 for IPv4 addresses
874 if not "/" in network and not ":" in network:
875 network = "%s/24" % network
876
877 # Convert AS number to integer
878 autnum = int(autnum)
879
880 log.info("Found announcement for %s by %s" % (network, autnum))
881
882 self.db.execute("INSERT INTO announcements(network, autnum) \
883 VALUES(%s, %s) ON CONFLICT (network) DO \
884 UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
885 network, autnum,
886 )
887
888 log.info("Finished reading the %s routing table" % protocol)
889
890 def _bird_cmd(self, socket_path, command):
891 # Connect to the socket
892 s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
893 s.connect(socket_path)
894
895 # Allocate some buffer
896 buffer = b""
897
898 # Send the command
899 s.send(b"%s\n" % command.encode())
900
901 while True:
902 # Fill up the buffer
903 buffer += s.recv(4096)
904
905 while True:
906 # Search for the next newline
907 pos = buffer.find(b"\n")
908
909 # If we cannot find one, we go back and read more data
910 if pos <= 0:
911 break
912
913 # Cut after the newline character
914 pos += 1
915
916 # Split the line we want and keep the rest in buffer
917 line, buffer = buffer[:pos], buffer[pos:]
918
919 # Look for the end-of-output indicator
920 if line == b"0000 \n":
921 return
922
923 # Otherwise return the line
924 yield line
925
926 def handle_update_overrides(self, ns):
927 with self.db.transaction():
928 # Drop all data that we have
929 self.db.execute("""
930 TRUNCATE TABLE autnum_overrides;
931 TRUNCATE TABLE network_overrides;
932 """)
933
934 for file in ns.files:
935 log.info("Reading %s..." % file)
936
937 with open(file, "rb") as f:
938 for type, block in location.importer.read_blocks(f):
939 if type == "net":
940 network = block.get("net")
941 # Try to parse and normalise the network
942 try:
943 network = ipaddress.ip_network(network, strict=False)
944 except ValueError as e:
945 log.warning("Invalid IP network: %s: %s" % (network, e))
946 continue
947
948 # Prevent that we overwrite all networks
949 if network.prefixlen == 0:
950 log.warning("Skipping %s: You cannot overwrite default" % network)
951 continue
952
953 self.db.execute("""
954 INSERT INTO network_overrides(
955 network,
956 country,
957 is_anonymous_proxy,
958 is_satellite_provider,
959 is_anycast
960 ) VALUES (%s, %s, %s, %s, %s)
961 ON CONFLICT (network) DO NOTHING""",
962 "%s" % network,
963 block.get("country"),
964 self._parse_bool(block, "is-anonymous-proxy"),
965 self._parse_bool(block, "is-satellite-provider"),
966 self._parse_bool(block, "is-anycast"),
967 )
968
969 elif type == "aut-num":
970 autnum = block.get("aut-num")
971
972 # Check if AS number begins with "AS"
973 if not autnum.startswith("AS"):
974 log.warning("Invalid AS number: %s" % autnum)
975 continue
976
977 # Strip "AS"
978 autnum = autnum[2:]
979
980 self.db.execute("""
981 INSERT INTO autnum_overrides(
982 number,
983 name,
984 country,
985 is_anonymous_proxy,
986 is_satellite_provider,
987 is_anycast
988 ) VALUES(%s, %s, %s, %s, %s, %s)
989 ON CONFLICT DO NOTHING""",
990 autnum,
991 block.get("name"),
992 block.get("country"),
993 self._parse_bool(block, "is-anonymous-proxy"),
994 self._parse_bool(block, "is-satellite-provider"),
995 self._parse_bool(block, "is-anycast"),
996 )
997
998 else:
999 log.warning("Unsupported type: %s" % type)
1000
1001 @staticmethod
1002 def _parse_bool(block, key):
1003 val = block.get(key)
1004
1005 # There is no point to proceed when we got None
1006 if val is None:
1007 return
1008
1009 # Convert to lowercase
1010 val = val.lower()
1011
1012 # True
1013 if val in ("yes", "1"):
1014 return True
1015
1016 # False
1017 if val in ("no", "0"):
1018 return False
1019
1020 # Default to None
1021 return None
1022
1023 def handle_import_countries(self, ns):
1024 with self.db.transaction():
1025 # Drop all data that we have
1026 self.db.execute("TRUNCATE TABLE countries")
1027
1028 for file in ns.file:
1029 for line in file:
1030 line = line.rstrip()
1031
1032 # Ignore any comments
1033 if line.startswith("#"):
1034 continue
1035
1036 try:
1037 country_code, continent_code, name = line.split(maxsplit=2)
1038 except:
1039 log.warning("Could not parse line: %s" % line)
1040 continue
1041
1042 self.db.execute("INSERT INTO countries(country_code, name, continent_code) \
1043 VALUES(%s, %s, %s) ON CONFLICT DO NOTHING", country_code, name, continent_code)
1044
1045
1046 def split_line(line):
1047 key, colon, val = line.partition(":")
1048
1049 # Strip any excess space
1050 key = key.strip()
1051 val = val.strip()
1052
1053 return key, val
1054
1055 def main():
1056 # Run the command line interface
1057 c = CLI()
1058 c.run()
1059
1060 main()
A library to determine the location of someone on the Internet