2 ###############################################################################
4 # libloc - A library to determine the location of someone on the Internet #
6 # Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
8 # This library is free software; you can redistribute it and/or #
9 # modify it under the terms of the GNU Lesser General Public #
10 # License as published by the Free Software Foundation; either #
11 # version 2.1 of the License, or (at your option) any later version. #
13 # This library is distributed in the hope that it will be useful, #
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
16 # Lesser General Public License for more details. #
18 ###############################################################################
28 # Load our location module
30 import location
.database
31 import location
.importer
32 from location
.i18n
import _
35 log
= logging
.getLogger("location.importer")
40 parser
= argparse
.ArgumentParser(
41 description
=_("Location Importer Command Line Interface"),
43 subparsers
= parser
.add_subparsers()
45 # Global configuration flags
46 parser
.add_argument("--debug", action
="store_true",
47 help=_("Enable debug output"))
48 parser
.add_argument("--quiet", action
="store_true",
49 help=_("Enable quiet mode"))
52 parser
.add_argument("--version", action
="version",
53 version
="%(prog)s @VERSION@")
56 parser
.add_argument("--database-host", required
=True,
57 help=_("Database Hostname"), metavar
=_("HOST"))
58 parser
.add_argument("--database-name", required
=True,
59 help=_("Database Name"), metavar
=_("NAME"))
60 parser
.add_argument("--database-username", required
=True,
61 help=_("Database Username"), metavar
=_("USERNAME"))
62 parser
.add_argument("--database-password", required
=True,
63 help=_("Database Password"), metavar
=_("PASSWORD"))
66 write
= subparsers
.add_parser("write", help=_("Write database to file"))
67 write
.set_defaults(func
=self
.handle_write
)
68 write
.add_argument("file", nargs
=1, help=_("Database File"))
69 write
.add_argument("--signing-key", nargs
="?", type=open, help=_("Signing Key"))
70 write
.add_argument("--backup-signing-key", nargs
="?", type=open, help=_("Backup Signing Key"))
71 write
.add_argument("--vendor", nargs
="?", help=_("Sets the vendor"))
72 write
.add_argument("--description", nargs
="?", help=_("Sets a description"))
73 write
.add_argument("--license", nargs
="?", help=_("Sets the license"))
74 write
.add_argument("--version", type=int, help=_("Database Format Version"))
77 update_whois
= subparsers
.add_parser("update-whois", help=_("Update WHOIS Information"))
78 update_whois
.set_defaults(func
=self
.handle_update_whois
)
80 # Update announcements
81 update_announcements
= subparsers
.add_parser("update-announcements",
82 help=_("Update BGP Annoucements"))
83 update_announcements
.set_defaults(func
=self
.handle_update_announcements
)
84 update_announcements
.add_argument("server", nargs
=1,
85 help=_("Route Server to connect to"), metavar
=_("SERVER"))
88 update_overrides
= subparsers
.add_parser("update-overrides",
89 help=_("Update overrides"),
91 update_overrides
.add_argument(
92 "files", nargs
="+", help=_("Files to import"),
94 update_overrides
.set_defaults(func
=self
.handle_update_overrides
)
97 import_countries
= subparsers
.add_parser("import-countries",
98 help=_("Import countries"),
100 import_countries
.add_argument("file", nargs
=1, type=argparse
.FileType("r"),
101 help=_("File to import"))
102 import_countries
.set_defaults(func
=self
.handle_import_countries
)
104 args
= parser
.parse_args()
108 location
.logger
.set_level(logging
.DEBUG
)
110 location
.logger
.set_level(logging
.WARNING
)
112 # Print usage if no action was given
113 if not "func" in args
:
120 # Parse command line arguments
121 args
= self
.parse_cli()
123 # Initialise database
124 self
.db
= self
._setup
_database
(args
)
127 ret
= args
.func(args
)
129 # Return with exit code
133 # Otherwise just exit
136 def _setup_database(self
, ns
):
138 Initialise the database
140 # Connect to database
141 db
= location
.database
.Connection(
142 host
=ns
.database_host
, database
=ns
.database_name
,
143 user
=ns
.database_username
, password
=ns
.database_password
,
146 with db
.transaction():
149 CREATE TABLE IF NOT EXISTS announcements(network inet, autnum bigint,
150 first_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP,
151 last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
152 CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
153 CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
156 CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
157 CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number);
160 CREATE TABLE IF NOT EXISTS countries(
161 country_code text NOT NULL, name text NOT NULL, continent_code text NOT NULL);
162 CREATE UNIQUE INDEX IF NOT EXISTS countries_country_code ON countries(country_code);
165 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
166 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
167 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
170 CREATE TABLE IF NOT EXISTS autnum_overrides(
171 number bigint NOT NULL,
174 is_anonymous_proxy boolean DEFAULT FALSE,
175 is_satellite_provider boolean DEFAULT FALSE,
176 is_anycast boolean DEFAULT FALSE
178 CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number
179 ON autnum_overrides(number);
181 CREATE TABLE IF NOT EXISTS network_overrides(
182 network inet NOT NULL,
184 is_anonymous_proxy boolean DEFAULT FALSE,
185 is_satellite_provider boolean DEFAULT FALSE,
186 is_anycast boolean DEFAULT FALSE
188 CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
189 ON network_overrides(network);
194 def handle_write(self
, ns
):
196 Compiles a database in libloc format out of what is in the database
199 writer
= location
.Writer(ns
.signing_key
, ns
.backup_signing_key
)
203 writer
.vendor
= ns
.vendor
206 writer
.description
= ns
.description
209 writer
.license
= ns
.license
211 # Add all Autonomous Systems
212 log
.info("Writing Autonomous Systems...")
214 # Select all ASes with a name
215 rows
= self
.db
.query("""
217 autnums.number AS number,
219 (SELECT overrides.name FROM autnum_overrides overrides
220 WHERE overrides.number = autnums.number),
224 WHERE name <> %s ORDER BY number
228 a
= writer
.add_as(row
.number
)
232 log
.info("Writing networks...")
234 # Select all known networks
235 rows
= self
.db
.query("""
237 DISTINCT ON (announcements.network)
238 announcements.network AS network,
239 announcements.autnum AS autnum,
244 SELECT country FROM network_overrides overrides
245 WHERE announcements.network <<= overrides.network
246 ORDER BY masklen(overrides.network) DESC
250 SELECT country FROM autnum_overrides overrides
251 WHERE announcements.autnum = overrides.number
256 -- Must be part of returned values for ORDER BY clause
257 masklen(networks.network) AS sort,
262 SELECT is_anonymous_proxy FROM network_overrides overrides
263 WHERE announcements.network <<= overrides.network
264 ORDER BY masklen(overrides.network) DESC
268 SELECT is_anonymous_proxy FROM autnum_overrides overrides
269 WHERE announcements.autnum = overrides.number
271 ) AS is_anonymous_proxy,
274 SELECT is_satellite_provider FROM network_overrides overrides
275 WHERE announcements.network <<= overrides.network
276 ORDER BY masklen(overrides.network) DESC
280 SELECT is_satellite_provider FROM autnum_overrides overrides
281 WHERE announcements.autnum = overrides.number
283 ) AS is_satellite_provider,
286 SELECT is_anycast FROM network_overrides overrides
287 WHERE announcements.network <<= overrides.network
288 ORDER BY masklen(overrides.network) DESC
292 SELECT is_anycast FROM autnum_overrides overrides
293 WHERE announcements.autnum = overrides.number
297 LEFT JOIN networks ON announcements.network <<= networks.network
298 ORDER BY announcements.network, sort DESC
302 network
= writer
.add_network(row
.network
)
305 network
.asn
, network
.country_code
= row
.autnum
, row
.country
308 if row
.is_anonymous_proxy
:
309 network
.set_flag(location
.NETWORK_FLAG_ANONYMOUS_PROXY
)
311 if row
.is_satellite_provider
:
312 network
.set_flag(location
.NETWORK_FLAG_SATELLITE_PROVIDER
)
315 network
.set_flag(location
.NETWORK_FLAG_ANYCAST
)
318 log
.info("Writing countries...")
319 rows
= self
.db
.query("SELECT * FROM countries ORDER BY country_code")
322 c
= writer
.add_country(row
.country_code
)
323 c
.continent_code
= row
.continent_code
326 # Write everything to file
327 log
.info("Writing database to file...")
331 def handle_update_whois(self
, ns
):
332 downloader
= location
.importer
.Downloader()
334 # Download all sources
335 with self
.db
.transaction():
336 # Create some temporary tables to store parsed data
338 CREATE TEMPORARY TABLE _autnums(number integer, organization text)
340 CREATE UNIQUE INDEX _autnums_number ON _autnums(number);
342 CREATE TEMPORARY TABLE _organizations(handle text, name text)
344 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
347 for source
in location
.importer
.WHOIS_SOURCES
:
348 with downloader
.request(source
, return_blocks
=True) as f
:
350 self
._parse
_block
(block
)
353 INSERT INTO autnums(number, name)
354 SELECT _autnums.number, _organizations.name FROM _autnums
355 LEFT JOIN _organizations ON _autnums.organization = _organizations.handle
356 ON CONFLICT (number) DO UPDATE SET name = excluded.name;
359 # Download all extended sources
360 for source
in location
.importer
.EXTENDED_SOURCES
:
361 with self
.db
.transaction():
363 with downloader
.request(source
) as f
:
365 self
._parse
_line
(line
)
367 def _parse_block(self
, block
):
368 # Get first line to find out what type of block this is
372 if line
.startswith("aut-num:"):
373 return self
._parse
_autnum
_block
(block
)
376 elif line
.startswith("organisation:"):
377 return self
._parse
_org
_block
(block
)
379 def _parse_autnum_block(self
, block
):
383 key
, val
= split_line(line
)
386 m
= re
.match(r
"^(AS|as)(\d+)", val
)
388 autnum
["asn"] = m
.group(2)
397 # Insert into database
398 self
.db
.execute("INSERT INTO _autnums(number, organization) \
399 VALUES(%s, %s) ON CONFLICT (number) DO UPDATE SET \
400 organization = excluded.organization",
401 autnum
.get("asn"), autnum
.get("org"),
404 def _parse_org_block(self
, block
):
408 key
, val
= split_line(line
)
410 if key
in ("organisation", "org-name"):
417 self
.db
.execute("INSERT INTO _organizations(handle, name) \
418 VALUES(%s, %s) ON CONFLICT (handle) DO \
419 UPDATE SET name = excluded.name",
420 org
.get("organisation"), org
.get("org-name"),
423 def _parse_line(self
, line
):
425 if line
.startswith("2"):
429 if line
.startswith("#"):
433 registry
, country_code
, type, line
= line
.split("|", 3)
435 log
.warning("Could not parse line: %s" % line
)
438 # Skip any lines that are for stats only
439 if country_code
== "*":
442 if type in ("ipv6", "ipv4"):
443 return self
._parse
_ip
_line
(country_code
, type, line
)
445 def _parse_ip_line(self
, country
, type, line
):
447 address
, prefix
, date
, status
, organization
= line
.split("|")
451 # Try parsing the line without organization
453 address
, prefix
, date
, status
= line
.split("|")
455 log
.warning("Unhandled line format: %s" % line
)
458 # Skip anything that isn't properly assigned
459 if not status
in ("assigned", "allocated"):
462 # Cast prefix into an integer
466 log
.warning("Invalid prefix: %s" % prefix
)
469 # Fix prefix length for IPv4
471 prefix
= 32 - int(math
.log(prefix
, 2))
473 # Try to parse the address
475 network
= ipaddress
.ip_network("%s/%s" % (address
, prefix
), strict
=False)
477 log
.warning("Invalid IP address: %s" % address
)
480 self
.db
.execute("INSERT INTO networks(network, country) \
481 VALUES(%s, %s) ON CONFLICT (network) DO \
482 UPDATE SET country = excluded.country",
483 "%s" % network
, country
,
486 def handle_update_announcements(self
, ns
):
487 server
= ns
.server
[0]
489 # Pre-compile regular expression for routes
490 route
= re
.compile(b
"^\*[\s\>]i([^\s]+).+?(\d+)\si\r\n", re
.MULTILINE|re
.DOTALL
)
492 with telnetlib
.Telnet(server
) as t
:
495 # t.set_debuglevel(10)
497 # Wait for console greeting
498 greeting
= t
.read_until(b
"> ", timeout
=30)
500 log
.error("Could not get a console prompt")
504 t
.write(b
"terminal length 0\n")
506 # Wait for the prompt to return
509 # Fetch the routing tables
510 with self
.db
.transaction():
511 for protocol
in ("ipv6", "ipv4"):
512 log
.info("Requesting %s routing table" % protocol
)
514 # Request the full unicast routing table
515 t
.write(b
"show bgp %s unicast\n" % protocol
.encode())
517 # Read entire header which ends with "Path"
518 t
.read_until(b
"Path\r\n")
521 # Try reading a full entry
522 # Those might be broken across multiple lines but ends with i
523 line
= t
.read_until(b
"i\r\n", timeout
=5)
527 # Show line for debugging
528 #log.debug(repr(line))
530 # Try finding a route in here
531 m
= route
.match(line
)
533 network
, autnum
= m
.groups()
535 # Convert network to string
536 network
= network
.decode()
538 # Append /24 for IPv4 addresses
539 if not "/" in network
and not ":" in network
:
540 network
= "%s/24" % network
542 # Convert AS number to integer
545 log
.info("Found announcement for %s by %s" % (network
, autnum
))
547 self
.db
.execute("INSERT INTO announcements(network, autnum) \
548 VALUES(%s, %s) ON CONFLICT (network) DO \
549 UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
553 log
.info("Finished reading the %s routing table" % protocol
)
555 # Purge anything we never want here
557 -- Delete default routes
558 DELETE FROM announcements WHERE network = '::/0' OR network = '0.0.0.0/0';
560 -- Delete anything that is not global unicast address space
561 DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3';
563 -- DELETE "current network" address space
564 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '0.0.0.0/8';
566 -- DELETE local loopback address space
567 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '127.0.0.0/8';
569 -- DELETE RFC 1918 address space
570 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8';
571 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12';
572 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16';
574 -- DELETE test, benchmark and documentation address space
575 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.0.0/24';
576 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.2.0/24';
577 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.18.0.0/15';
578 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.51.100.0/24';
579 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '203.0.113.0/24';
581 -- DELETE CGNAT address space (RFC 6598)
582 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '100.64.0.0/10';
584 -- DELETE link local address space
585 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '169.254.0.0/16';
587 -- DELETE IPv6 to IPv4 (6to4) address space
588 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.88.99.0/24';
590 -- DELETE multicast and reserved address space
591 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '224.0.0.0/4';
592 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '240.0.0.0/4';
594 -- Delete networks that are too small to be in the global routing table
595 DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48;
596 DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24;
598 -- Delete any non-public or reserved ASNs
599 DELETE FROM announcements WHERE NOT (
600 (autnum >= 1 AND autnum <= 23455)
602 (autnum >= 23457 AND autnum <= 64495)
604 (autnum >= 131072 AND autnum <= 4199999999)
607 -- Delete everything that we have not seen for 14 days
608 DELETE FROM announcements WHERE last_seen_at <= CURRENT_TIMESTAMP - INTERVAL '14 days';
611 def handle_update_overrides(self
, ns
):
612 with self
.db
.transaction():
613 # Drop all data that we have
615 TRUNCATE TABLE autnum_overrides;
616 TRUNCATE TABLE network_overrides;
619 for file in ns
.files
:
620 log
.info("Reading %s..." % file)
622 with
open(file, "rb") as f
:
623 for type, block
in location
.importer
.read_blocks(f
):
625 network
= block
.get("net")
626 # Try to parse and normalise the network
628 network
= ipaddress
.ip_network(network
, strict
=False)
629 except ValueError as e
:
630 log
.warning("Invalid IP network: %s: %s" % (network
, e
))
633 # Prevent that we overwrite all networks
634 if network
.prefixlen
== 0:
635 log
.warning("Skipping %s: You cannot overwrite default" % network
)
639 INSERT INTO network_overrides(
643 is_satellite_provider,
645 ) VALUES (%s, %s, %s, %s, %s)
646 ON CONFLICT (network) DO NOTHING""",
648 block
.get("country"),
649 block
.get("is-anonymous-proxy") == "yes",
650 block
.get("is-satellite-provider") == "yes",
651 block
.get("is-anycast") == "yes",
654 elif type == "aut-num":
655 autnum
= block
.get("aut-num")
657 # Check if AS number begins with "AS"
658 if not autnum
.startswith("AS"):
659 log
.warning("Invalid AS number: %s" % autnum
)
666 INSERT INTO autnum_overrides(
671 is_satellite_provider,
673 ) VALUES(%s, %s, %s, %s, %s, %s)
674 ON CONFLICT DO NOTHING""",
677 block
.get("country"),
678 block
.get("is-anonymous-proxy") == "yes",
679 block
.get("is-satellite-provider") == "yes",
680 block
.get("is-anycast") == "yes",
684 log
.warning("Unsupport type: %s" % type)
686 def handle_import_countries(self
, ns
):
687 with self
.db
.transaction():
688 # Drop all data that we have
689 self
.db
.execute("TRUNCATE TABLE countries")
695 # Ignore any comments
696 if line
.startswith("#"):
700 country_code
, continent_code
, name
= line
.split(maxsplit
=2)
702 log
.warning("Could not parse line: %s" % line
)
705 self
.db
.execute("INSERT INTO countries(country_code, name, continent_code) \
706 VALUES(%s, %s, %s) ON CONFLICT DO NOTHING", country_code
, name
, continent_code
)
709 def split_line(line
):
710 key
, colon
, val
= line
.partition(":")
712 # Strip any excess space
719 # Run the command line interface