]> git.ipfire.org Git - people/ms/libloc.git/blob - src/python/location-importer.in
projects / people / ms / libloc.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
importer: Drop EDROP as it has been merged into DROP
[people/ms/libloc.git] / src / python / location-importer.in
1 #!/usr/bin/python3
2 ###############################################################################
3 # #
4 # libloc - A library to determine the location of someone on the Internet #
5 # #
6 # Copyright (C) 2020-2021 IPFire Development Team <info@ipfire.org> #
7 # #
8 # This library is free software; you can redistribute it and/or #
9 # modify it under the terms of the GNU Lesser General Public #
10 # License as published by the Free Software Foundation; either #
11 # version 2.1 of the License, or (at your option) any later version. #
12 # #
13 # This library is distributed in the hope that it will be useful, #
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
16 # Lesser General Public License for more details. #
17 # #
18 ###############################################################################
19
20 import argparse
21 import ipaddress
22 import logging
23 import math
24 import re
25 import socket
26 import sys
27 import telnetlib
28
29 # Load our location module
30 import location
31 import location.database
32 import location.importer
33 from location.i18n import _
34
35 # Initialise logging
36 log = logging.getLogger("location.importer")
37 log.propagate = 1
38
39 class CLI(object):
40 def parse_cli(self):
41 parser = argparse.ArgumentParser(
42 description=_("Location Importer Command Line Interface"),
43 )
44 subparsers = parser.add_subparsers()
45
46 # Global configuration flags
47 parser.add_argument("--debug", action="store_true",
48 help=_("Enable debug output"))
49 parser.add_argument("--quiet", action="store_true",
50 help=_("Enable quiet mode"))
51
52 # version
53 parser.add_argument("--version", action="version",
54 version="%(prog)s @VERSION@")
55
56 # Database
57 parser.add_argument("--database-host", required=True,
58 help=_("Database Hostname"), metavar=_("HOST"))
59 parser.add_argument("--database-name", required=True,
60 help=_("Database Name"), metavar=_("NAME"))
61 parser.add_argument("--database-username", required=True,
62 help=_("Database Username"), metavar=_("USERNAME"))
63 parser.add_argument("--database-password", required=True,
64 help=_("Database Password"), metavar=_("PASSWORD"))
65
66 # Write Database
67 write = subparsers.add_parser("write", help=_("Write database to file"))
68 write.set_defaults(func=self.handle_write)
69 write.add_argument("file", nargs=1, help=_("Database File"))
70 write.add_argument("--signing-key", nargs="?", type=open, help=_("Signing Key"))
71 write.add_argument("--backup-signing-key", nargs="?", type=open, help=_("Backup Signing Key"))
72 write.add_argument("--vendor", nargs="?", help=_("Sets the vendor"))
73 write.add_argument("--description", nargs="?", help=_("Sets a description"))
74 write.add_argument("--license", nargs="?", help=_("Sets the license"))
75 write.add_argument("--version", type=int, help=_("Database Format Version"))
76
77 # Update WHOIS
78 update_whois = subparsers.add_parser("update-whois", help=_("Update WHOIS Information"))
79 update_whois.set_defaults(func=self.handle_update_whois)
80
81 # Update announcements
82 update_announcements = subparsers.add_parser("update-announcements",
83 help=_("Update BGP Annoucements"))
84 update_announcements.set_defaults(func=self.handle_update_announcements)
85 update_announcements.add_argument("server", nargs=1,
86 help=_("Route Server to connect to"), metavar=_("SERVER"))
87
88 # Update overrides
89 update_overrides = subparsers.add_parser("update-overrides",
90 help=_("Update overrides"),
91 )
92 update_overrides.add_argument(
93 "files", nargs="+", help=_("Files to import"),
94 )
95 update_overrides.set_defaults(func=self.handle_update_overrides)
96
97 # Import countries
98 import_countries = subparsers.add_parser("import-countries",
99 help=_("Import countries"),
100 )
101 import_countries.add_argument("file", nargs=1, type=argparse.FileType("r"),
102 help=_("File to import"))
103 import_countries.set_defaults(func=self.handle_import_countries)
104
105 args = parser.parse_args()
106
107 # Configure logging
108 if args.debug:
109 location.logger.set_level(logging.DEBUG)
110 elif args.quiet:
111 location.logger.set_level(logging.WARNING)
112
113 # Print usage if no action was given
114 if not "func" in args:
115 parser.print_usage()
116 sys.exit(2)
117
118 return args
119
120 def run(self):
121 # Parse command line arguments
122 args = self.parse_cli()
123
124 # Initialise database
125 self.db = self._setup_database(args)
126
127 # Call function
128 ret = args.func(args)
129
130 # Return with exit code
131 if ret:
132 sys.exit(ret)
133
134 # Otherwise just exit
135 sys.exit(0)
136
137 def _setup_database(self, ns):
138 """
139 Initialise the database
140 """
141 # Connect to database
142 db = location.database.Connection(
143 host=ns.database_host, database=ns.database_name,
144 user=ns.database_username, password=ns.database_password,
145 )
146
147 with db.transaction():
148 db.execute("""
149 -- announcements
150 CREATE TABLE IF NOT EXISTS announcements(network inet, autnum bigint,
151 first_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP,
152 last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
153 CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
154 CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
155 CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops);
156
157 -- autnums
158 CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
159 CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number);
160
161 -- countries
162 CREATE TABLE IF NOT EXISTS countries(
163 country_code text NOT NULL, name text NOT NULL, continent_code text NOT NULL);
164 CREATE UNIQUE INDEX IF NOT EXISTS countries_country_code ON countries(country_code);
165
166 -- networks
167 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
168 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
169 CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
170 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
171
172 -- overrides
173 CREATE TABLE IF NOT EXISTS autnum_overrides(
174 number bigint NOT NULL,
175 name text,
176 country text,
177 is_anonymous_proxy boolean,
178 is_satellite_provider boolean,
179 is_anycast boolean
180 );
181 CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number
182 ON autnum_overrides(number);
183
184 CREATE TABLE IF NOT EXISTS network_overrides(
185 network inet NOT NULL,
186 country text,
187 is_anonymous_proxy boolean,
188 is_satellite_provider boolean,
189 is_anycast boolean
190 );
191 CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
192 ON network_overrides(network);
193 CREATE INDEX IF NOT EXISTS network_overrides_search
194 ON network_overrides USING GIST(network inet_ops);
195 """)
196
197 return db
198
199 def handle_write(self, ns):
200 """
201 Compiles a database in libloc format out of what is in the database
202 """
203 # Allocate a writer
204 writer = location.Writer(ns.signing_key, ns.backup_signing_key)
205
206 # Set all metadata
207 if ns.vendor:
208 writer.vendor = ns.vendor
209
210 if ns.description:
211 writer.description = ns.description
212
213 if ns.license:
214 writer.license = ns.license
215
216 # Add all Autonomous Systems
217 log.info("Writing Autonomous Systems...")
218
219 # Select all ASes with a name
220 rows = self.db.query("""
221 SELECT
222 autnums.number AS number,
223 COALESCE(
224 (SELECT overrides.name FROM autnum_overrides overrides
225 WHERE overrides.number = autnums.number),
226 autnums.name
227 ) AS name
228 FROM autnums
229 WHERE name <> %s ORDER BY number
230 """, "")
231
232 for row in rows:
233 a = writer.add_as(row.number)
234 a.name = row.name
235
236 # Add all networks
237 log.info("Writing networks...")
238
239 # Select all known networks
240 rows = self.db.query("""
241 -- Return a list of those networks enriched with all
242 -- other information that we store in the database
243 SELECT
244 DISTINCT ON (network)
245 network,
246 autnum,
247
248 -- Country
249 COALESCE(
250 (
251 SELECT country FROM network_overrides overrides
252 WHERE networks.network <<= overrides.network
253 ORDER BY masklen(overrides.network) DESC
254 LIMIT 1
255 ),
256 (
257 SELECT country FROM autnum_overrides overrides
258 WHERE networks.autnum = overrides.number
259 ),
260 networks.country
261 ) AS country,
262
263 -- Flags
264 COALESCE(
265 (
266 SELECT is_anonymous_proxy FROM network_overrides overrides
267 WHERE networks.network <<= overrides.network
268 ORDER BY masklen(overrides.network) DESC
269 LIMIT 1
270 ),
271 (
272 SELECT is_anonymous_proxy FROM autnum_overrides overrides
273 WHERE networks.autnum = overrides.number
274 ),
275 FALSE
276 ) AS is_anonymous_proxy,
277 COALESCE(
278 (
279 SELECT is_satellite_provider FROM network_overrides overrides
280 WHERE networks.network <<= overrides.network
281 ORDER BY masklen(overrides.network) DESC
282 LIMIT 1
283 ),
284 (
285 SELECT is_satellite_provider FROM autnum_overrides overrides
286 WHERE networks.autnum = overrides.number
287 ),
288 FALSE
289 ) AS is_satellite_provider,
290 COALESCE(
291 (
292 SELECT is_anycast FROM network_overrides overrides
293 WHERE networks.network <<= overrides.network
294 ORDER BY masklen(overrides.network) DESC
295 LIMIT 1
296 ),
297 (
298 SELECT is_anycast FROM autnum_overrides overrides
299 WHERE networks.autnum = overrides.number
300 ),
301 FALSE
302 ) AS is_anycast
303 FROM (
304 SELECT
305 known_networks.network AS network,
306 announcements.autnum AS autnum,
307 networks.country AS country,
308
309 -- Must be part of returned values for ORDER BY clause
310 masklen(announcements.network) AS sort_a,
311 masklen(networks.network) AS sort_b
312 FROM (
313 SELECT network FROM announcements
314 UNION ALL
315 SELECT network FROM networks
316 UNION ALL
317 SELECT network FROM network_overrides
318 ) known_networks
319 LEFT JOIN
320 announcements ON known_networks.network <<= announcements.network
321 LEFT JOIN
322 networks ON known_networks.network <<= networks.network
323 ORDER BY
324 known_networks.network,
325 sort_a DESC,
326 sort_b DESC
327 ) networks
328 """)
329
330 for row in rows:
331 network = writer.add_network(row.network)
332
333 # Save country
334 if row.country:
335 network.country_code = row.country
336
337 # Save ASN
338 if row.autnum:
339 network.asn = row.autnum
340
341 # Set flags
342 if row.is_anonymous_proxy:
343 network.set_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY)
344
345 if row.is_satellite_provider:
346 network.set_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER)
347
348 if row.is_anycast:
349 network.set_flag(location.NETWORK_FLAG_ANYCAST)
350
351 # Add all countries
352 log.info("Writing countries...")
353 rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
354
355 for row in rows:
356 c = writer.add_country(row.country_code)
357 c.continent_code = row.continent_code
358 c.name = row.name
359
360 # Write everything to file
361 log.info("Writing database to file...")
362 for file in ns.file:
363 writer.write(file)
364
365 def handle_update_whois(self, ns):
366 downloader = location.importer.Downloader()
367
368 # Download all sources
369 with self.db.transaction():
370 # Create some temporary tables to store parsed data
371 self.db.execute("""
372 CREATE TEMPORARY TABLE _autnums(number integer, organization text)
373 ON COMMIT DROP;
374 CREATE UNIQUE INDEX _autnums_number ON _autnums(number);
375
376 CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
377 ON COMMIT DROP;
378 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
379
380 CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
381 ON COMMIT DROP;
382 CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
383 CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
384 """)
385
386 # Remove all previously imported content
387 self.db.execute("""
388 TRUNCATE TABLE networks;
389 """)
390
391 # Fetch all valid country codes to check parsed networks aganist...
392 rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
393 validcountries = []
394
395 for row in rows:
396 validcountries.append(row.country_code)
397
398 for source in location.importer.WHOIS_SOURCES:
399 with downloader.request(source, return_blocks=True) as f:
400 for block in f:
401 self._parse_block(block, validcountries)
402
403 # Process all parsed networks from every RIR we happen to have access to,
404 # insert the largest network chunks into the networks table immediately...
405 families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
406
407 for family in (row.family for row in families):
408 smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
409
410 self.db.execute("INSERT INTO networks(network, country) \
411 SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
412
413 # ... determine any other prefixes for this network family, ...
414 prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
415 WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
416
417 # ... and insert networks with this prefix in case they provide additional
418 # information (i. e. subnet of a larger chunk with a different country)
419 for prefix in (row.prefix for row in prefixes):
420 self.db.execute("""
421 WITH candidates AS (
422 SELECT
423 _rirdata.network,
424 _rirdata.country
425 FROM
426 _rirdata
427 WHERE
428 family(_rirdata.network) = %s
429 AND
430 masklen(_rirdata.network) = %s
431 ),
432 filtered AS (
433 SELECT
434 DISTINCT ON (c.network)
435 c.network,
436 c.country,
437 masklen(networks.network),
438 networks.country AS parent_country
439 FROM
440 candidates c
441 LEFT JOIN
442 networks
443 ON
444 c.network << networks.network
445 ORDER BY
446 c.network,
447 masklen(networks.network) DESC NULLS LAST
448 )
449 INSERT INTO
450 networks(network, country)
451 SELECT
452 network,
453 country
454 FROM
455 filtered
456 WHERE
457 parent_country IS NULL
458 OR
459 country <> parent_country
460 ON CONFLICT DO NOTHING""",
461 family, prefix,
462 )
463
464 self.db.execute("""
465 INSERT INTO autnums(number, name)
466 SELECT _autnums.number, _organizations.name FROM _autnums
467 JOIN _organizations ON _autnums.organization = _organizations.handle
468 ON CONFLICT (number) DO UPDATE SET name = excluded.name;
469 """)
470
471 # Download all extended sources
472 for source in location.importer.EXTENDED_SOURCES:
473 with self.db.transaction():
474 # Download data
475 with downloader.request(source) as f:
476 for line in f:
477 self._parse_line(line, validcountries)
478
479 def _check_parsed_network(self, network):
480 """
481 Assistive function to detect and subsequently sort out parsed
482 networks from RIR data (both Whois and so-called "extended sources"),
483 which are or have...
484
485 (a) not globally routable (RFC 1918 space, et al.)
486 (b) covering a too large chunk of the IP address space (prefix length
487 is < 7 for IPv4 networks, and < 10 for IPv6)
488 (c) "0.0.0.0" or "::" as a network address
489 (d) are too small for being publicly announced (we have decided not to
490 process them at the moment, as they significantly enlarge our
491 database without providing very helpful additional information)
492
493 This unfortunately is necessary due to brain-dead clutter across
494 various RIR databases, causing mismatches and eventually disruptions.
495
496 We will return False in case a network is not suitable for adding
497 it to our database, and True otherwise.
498 """
499
500 if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)):
501 return False
502
503 if not network.is_global:
504 log.debug("Skipping non-globally routable network: %s" % network)
505 return False
506
507 if network.version == 4:
508 if network.prefixlen < 7:
509 log.debug("Skipping too big IP chunk: %s" % network)
510 return False
511
512 if network.prefixlen > 24:
513 log.debug("Skipping network too small to be publicly announced: %s" % network)
514 return False
515
516 if str(network.network_address) == "0.0.0.0":
517 log.debug("Skipping network based on 0.0.0.0: %s" % network)
518 return False
519
520 elif network.version == 6:
521 if network.prefixlen < 10:
522 log.debug("Skipping too big IP chunk: %s" % network)
523 return False
524
525 if network.prefixlen > 48:
526 log.debug("Skipping network too small to be publicly announced: %s" % network)
527 return False
528
529 if str(network.network_address) == "::":
530 log.debug("Skipping network based on '::': %s" % network)
531 return False
532
533 else:
534 # This should not happen...
535 log.warning("Skipping network of unknown family, this should not happen: %s" % network)
536 return False
537
538 # In case we have made it here, the network is considered to
539 # be suitable for libloc consumption...
540 return True
541
542 def _parse_block(self, block, validcountries = None):
543 # Get first line to find out what type of block this is
544 line = block[0]
545
546 # aut-num
547 if line.startswith("aut-num:"):
548 return self._parse_autnum_block(block)
549
550 # inetnum
551 if line.startswith("inet6num:") or line.startswith("inetnum:"):
552 return self._parse_inetnum_block(block, validcountries)
553
554 # organisation
555 elif line.startswith("organisation:"):
556 return self._parse_org_block(block)
557
558 def _parse_autnum_block(self, block):
559 autnum = {}
560 for line in block:
561 # Split line
562 key, val = split_line(line)
563
564 if key == "aut-num":
565 m = re.match(r"^(AS|as)(\d+)", val)
566 if m:
567 autnum["asn"] = m.group(2)
568
569 elif key == "org":
570 autnum[key] = val.upper()
571
572 # Skip empty objects
573 if not autnum:
574 return
575
576 # Insert into database
577 self.db.execute("INSERT INTO _autnums(number, organization) \
578 VALUES(%s, %s) ON CONFLICT (number) DO UPDATE SET \
579 organization = excluded.organization",
580 autnum.get("asn"), autnum.get("org"),
581 )
582
583 def _parse_inetnum_block(self, block, validcountries = None):
584 log.debug("Parsing inetnum block:")
585
586 inetnum = {}
587 for line in block:
588 log.debug(line)
589
590 # Split line
591 key, val = split_line(line)
592
593 # Filter any inetnum records which are only referring to IP space
594 # not managed by that specific RIR...
595 if key == "netname":
596 if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()):
597 log.debug("Skipping record indicating historic/orphaned data: %s" % val.strip())
598 return
599
600 if key == "inetnum":
601 start_address, delim, end_address = val.partition("-")
602
603 # Strip any excess space
604 start_address, end_address = start_address.rstrip(), end_address.strip()
605
606 # Convert to IP address
607 try:
608 start_address = ipaddress.ip_address(start_address)
609 end_address = ipaddress.ip_address(end_address)
610 except ValueError:
611 log.warning("Could not parse line: %s" % line)
612 return
613
614 inetnum["inetnum"] = list(ipaddress.summarize_address_range(start_address, end_address))
615
616 elif key == "inet6num":
617 inetnum[key] = [ipaddress.ip_network(val, strict=False)]
618
619 elif key == "country":
620 inetnum[key] = val.upper()
621
622 # Skip empty objects
623 if not inetnum or not "country" in inetnum:
624 return
625
626 # Iterate through all networks enumerated from above, check them for plausibility and insert
627 # them into the database, if _check_parsed_network() succeeded
628 for single_network in inetnum.get("inet6num") or inetnum.get("inetnum"):
629 if self._check_parsed_network(single_network):
630
631 # Skip objects with unknown country codes - to avoid log spam for invalid or too small
632 # networks, this check is - kinda ugly - done at this point
633 if validcountries and inetnum.get("country") not in validcountries:
634 log.warning("Skipping network with bogus country '%s': %s" % \
635 (inetnum.get("country"), inetnum.get("inet6num") or inetnum.get("inetnum")))
636 break
637
638 # Everything is fine here, run INSERT statement...
639 self.db.execute("INSERT INTO _rirdata(network, country) \
640 VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
641 "%s" % single_network, inetnum.get("country"),
642 )
643
644 def _parse_org_block(self, block):
645 org = {}
646 for line in block:
647 # Split line
648 key, val = split_line(line)
649
650 if key == "organisation":
651 org[key] = val.upper()
652 elif key == "org-name":
653 org[key] = val
654
655 # Skip empty objects
656 if not org:
657 return
658
659 self.db.execute("INSERT INTO _organizations(handle, name) \
660 VALUES(%s, %s) ON CONFLICT (handle) DO \
661 UPDATE SET name = excluded.name",
662 org.get("organisation"), org.get("org-name"),
663 )
664
665 def _parse_line(self, line, validcountries = None):
666 # Skip version line
667 if line.startswith("2"):
668 return
669
670 # Skip comments
671 if line.startswith("#"):
672 return
673
674 try:
675 registry, country_code, type, line = line.split("|", 3)
676 except:
677 log.warning("Could not parse line: %s" % line)
678 return
679
680 # Skip any lines that are for stats only or do not have a country
681 # code at all (avoids log spam below)
682 if not country_code or country_code == '*':
683 return
684
685 # Skip objects with unknown country codes
686 if validcountries and country_code not in validcountries:
687 log.warning("Skipping line with bogus country '%s': %s" % \
688 (country_code, line))
689 return
690
691 if type in ("ipv6", "ipv4"):
692 return self._parse_ip_line(country_code, type, line)
693
694 def _parse_ip_line(self, country, type, line):
695 try:
696 address, prefix, date, status, organization = line.split("|")
697 except ValueError:
698 organization = None
699
700 # Try parsing the line without organization
701 try:
702 address, prefix, date, status = line.split("|")
703 except ValueError:
704 log.warning("Unhandled line format: %s" % line)
705 return
706
707 # Skip anything that isn't properly assigned
708 if not status in ("assigned", "allocated"):
709 return
710
711 # Cast prefix into an integer
712 try:
713 prefix = int(prefix)
714 except:
715 log.warning("Invalid prefix: %s" % prefix)
716 return
717
718 # Fix prefix length for IPv4
719 if type == "ipv4":
720 prefix = 32 - int(math.log(prefix, 2))
721
722 # Try to parse the address
723 try:
724 network = ipaddress.ip_network("%s/%s" % (address, prefix), strict=False)
725 except ValueError:
726 log.warning("Invalid IP address: %s" % address)
727 return
728
729 if not self._check_parsed_network(network):
730 return
731
732 self.db.execute("INSERT INTO networks(network, country) \
733 VALUES(%s, %s) ON CONFLICT (network) DO \
734 UPDATE SET country = excluded.country",
735 "%s" % network, country,
736 )
737
738 def handle_update_announcements(self, ns):
739 server = ns.server[0]
740
741 with self.db.transaction():
742 if server.startswith("/"):
743 self._handle_update_announcements_from_bird(server)
744 else:
745 self._handle_update_announcements_from_telnet(server)
746
747 # Purge anything we never want here
748 self.db.execute("""
749 -- Delete default routes
750 DELETE FROM announcements WHERE network = '::/0' OR network = '0.0.0.0/0';
751
752 -- Delete anything that is not global unicast address space
753 DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3';
754
755 -- DELETE "current network" address space
756 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '0.0.0.0/8';
757
758 -- DELETE local loopback address space
759 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '127.0.0.0/8';
760
761 -- DELETE RFC 1918 address space
762 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8';
763 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12';
764 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16';
765
766 -- DELETE test, benchmark and documentation address space
767 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.0.0/24';
768 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.2.0/24';
769 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.18.0.0/15';
770 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.51.100.0/24';
771 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '203.0.113.0/24';
772
773 -- DELETE CGNAT address space (RFC 6598)
774 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '100.64.0.0/10';
775
776 -- DELETE link local address space
777 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '169.254.0.0/16';
778
779 -- DELETE IPv6 to IPv4 (6to4) address space (RFC 3068)
780 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.88.99.0/24';
781 DELETE FROM announcements WHERE family(network) = 6 AND network <<= '2002::/16';
782
783 -- DELETE multicast and reserved address space
784 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '224.0.0.0/4';
785 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '240.0.0.0/4';
786
787 -- Delete networks that are too small to be in the global routing table
788 DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48;
789 DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24;
790
791 -- Delete any non-public or reserved ASNs
792 DELETE FROM announcements WHERE NOT (
793 (autnum >= 1 AND autnum <= 23455)
794 OR
795 (autnum >= 23457 AND autnum <= 64495)
796 OR
797 (autnum >= 131072 AND autnum <= 4199999999)
798 );
799
800 -- Delete everything that we have not seen for 14 days
801 DELETE FROM announcements WHERE last_seen_at <= CURRENT_TIMESTAMP - INTERVAL '14 days';
802 """)
803
804 def _handle_update_announcements_from_bird(self, server):
805 # Pre-compile the regular expression for faster searching
806 route = re.compile(b"^\s(.+?)\s+.+?\[AS(.*?).\]$")
807
808 log.info("Requesting routing table from Bird (%s)" % server)
809
810 # Send command to list all routes
811 for line in self._bird_cmd(server, "show route"):
812 m = route.match(line)
813 if not m:
814 log.debug("Could not parse line: %s" % line.decode())
815 continue
816
817 # Fetch the extracted network and ASN
818 network, autnum = m.groups()
819
820 # Insert it into the database
821 self.db.execute("INSERT INTO announcements(network, autnum) \
822 VALUES(%s, %s) ON CONFLICT (network) DO \
823 UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
824 network.decode(), autnum.decode(),
825 )
826
827 def _handle_update_announcements_from_telnet(self, server):
828 # Pre-compile regular expression for routes
829 route = re.compile(b"^\*[\s\>]i([^\s]+).+?(\d+)\si\r\n", re.MULTILINE|re.DOTALL)
830
831 with telnetlib.Telnet(server) as t:
832 # Enable debug mode
833 #if ns.debug:
834 # t.set_debuglevel(10)
835
836 # Wait for console greeting
837 greeting = t.read_until(b"> ", timeout=30)
838 if not greeting:
839 log.error("Could not get a console prompt")
840 return 1
841
842 # Disable pagination
843 t.write(b"terminal length 0\n")
844
845 # Wait for the prompt to return
846 t.read_until(b"> ")
847
848 # Fetch the routing tables
849 for protocol in ("ipv6", "ipv4"):
850 log.info("Requesting %s routing table" % protocol)
851
852 # Request the full unicast routing table
853 t.write(b"show bgp %s unicast\n" % protocol.encode())
854
855 # Read entire header which ends with "Path"
856 t.read_until(b"Path\r\n")
857
858 while True:
859 # Try reading a full entry
860 # Those might be broken across multiple lines but ends with i
861 line = t.read_until(b"i\r\n", timeout=5)
862 if not line:
863 break
864
865 # Show line for debugging
866 #log.debug(repr(line))
867
868 # Try finding a route in here
869 m = route.match(line)
870 if m:
871 network, autnum = m.groups()
872
873 # Convert network to string
874 network = network.decode()
875
876 # Append /24 for IPv4 addresses
877 if not "/" in network and not ":" in network:
878 network = "%s/24" % network
879
880 # Convert AS number to integer
881 autnum = int(autnum)
882
883 log.info("Found announcement for %s by %s" % (network, autnum))
884
885 self.db.execute("INSERT INTO announcements(network, autnum) \
886 VALUES(%s, %s) ON CONFLICT (network) DO \
887 UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
888 network, autnum,
889 )
890
891 log.info("Finished reading the %s routing table" % protocol)
892
893 def _bird_cmd(self, socket_path, command):
894 # Connect to the socket
895 s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
896 s.connect(socket_path)
897
898 # Allocate some buffer
899 buffer = b""
900
901 # Send the command
902 s.send(b"%s\n" % command.encode())
903
904 while True:
905 # Fill up the buffer
906 buffer += s.recv(4096)
907
908 while True:
909 # Search for the next newline
910 pos = buffer.find(b"\n")
911
912 # If we cannot find one, we go back and read more data
913 if pos <= 0:
914 break
915
916 # Cut after the newline character
917 pos += 1
918
919 # Split the line we want and keep the rest in buffer
920 line, buffer = buffer[:pos], buffer[pos:]
921
922 # Look for the end-of-output indicator
923 if line == b"0000 \n":
924 return
925
926 # Otherwise return the line
927 yield line
928
929 def handle_update_overrides(self, ns):
930 with self.db.transaction():
931 # Drop all data that we have
932 self.db.execute("""
933 TRUNCATE TABLE autnum_overrides;
934 TRUNCATE TABLE network_overrides;
935 """)
936
937 for file in ns.files:
938 log.info("Reading %s..." % file)
939
940 with open(file, "rb") as f:
941 for type, block in location.importer.read_blocks(f):
942 if type == "net":
943 network = block.get("net")
944 # Try to parse and normalise the network
945 try:
946 network = ipaddress.ip_network(network, strict=False)
947 except ValueError as e:
948 log.warning("Invalid IP network: %s: %s" % (network, e))
949 continue
950
951 # Prevent that we overwrite all networks
952 if network.prefixlen == 0:
953 log.warning("Skipping %s: You cannot overwrite default" % network)
954 continue
955
956 self.db.execute("""
957 INSERT INTO network_overrides(
958 network,
959 country,
960 is_anonymous_proxy,
961 is_satellite_provider,
962 is_anycast
963 ) VALUES (%s, %s, %s, %s, %s)
964 ON CONFLICT (network) DO NOTHING""",
965 "%s" % network,
966 block.get("country"),
967 self._parse_bool(block, "is-anonymous-proxy"),
968 self._parse_bool(block, "is-satellite-provider"),
969 self._parse_bool(block, "is-anycast"),
970 )
971
972 elif type == "aut-num":
973 autnum = block.get("aut-num")
974
975 # Check if AS number begins with "AS"
976 if not autnum.startswith("AS"):
977 log.warning("Invalid AS number: %s" % autnum)
978 continue
979
980 # Strip "AS"
981 autnum = autnum[2:]
982
983 self.db.execute("""
984 INSERT INTO autnum_overrides(
985 number,
986 name,
987 country,
988 is_anonymous_proxy,
989 is_satellite_provider,
990 is_anycast
991 ) VALUES(%s, %s, %s, %s, %s, %s)
992 ON CONFLICT DO NOTHING""",
993 autnum,
994 block.get("name"),
995 block.get("country"),
996 self._parse_bool(block, "is-anonymous-proxy"),
997 self._parse_bool(block, "is-satellite-provider"),
998 self._parse_bool(block, "is-anycast"),
999 )
1000
1001 else:
1002 log.warning("Unsupported type: %s" % type)
1003
1004 @staticmethod
1005 def _parse_bool(block, key):
1006 val = block.get(key)
1007
1008 # There is no point to proceed when we got None
1009 if val is None:
1010 return
1011
1012 # Convert to lowercase
1013 val = val.lower()
1014
1015 # True
1016 if val in ("yes", "1"):
1017 return True
1018
1019 # False
1020 if val in ("no", "0"):
1021 return False
1022
1023 # Default to None
1024 return None
1025
1026 def handle_import_countries(self, ns):
1027 with self.db.transaction():
1028 # Drop all data that we have
1029 self.db.execute("TRUNCATE TABLE countries")
1030
1031 for file in ns.file:
1032 for line in file:
1033 line = line.rstrip()
1034
1035 # Ignore any comments
1036 if line.startswith("#"):
1037 continue
1038
1039 try:
1040 country_code, continent_code, name = line.split(maxsplit=2)
1041 except:
1042 log.warning("Could not parse line: %s" % line)
1043 continue
1044
1045 self.db.execute("INSERT INTO countries(country_code, name, continent_code) \
1046 VALUES(%s, %s, %s) ON CONFLICT DO NOTHING", country_code, name, continent_code)
1047
1048
1049 def split_line(line):
1050 key, colon, val = line.partition(":")
1051
1052 # Strip any excess space
1053 key = key.strip()
1054 val = val.strip()
1055
1056 return key, val
1057
1058 def main():
1059 # Run the command line interface
1060 c = CLI()
1061 c.run()
1062
1063 main()
A library to determine the location of someone on the Internet