2 ###############################################################################
4 # libloc - A library to determine the location of someone on the Internet #
6 # Copyright (C) 2020-2021 IPFire Development Team <info@ipfire.org> #
8 # This library is free software; you can redistribute it and/or #
9 # modify it under the terms of the GNU Lesser General Public #
10 # License as published by the Free Software Foundation; either #
11 # version 2.1 of the License, or (at your option) any later version. #
13 # This library is distributed in the hope that it will be useful, #
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
16 # Lesser General Public License for more details. #
18 ###############################################################################
29 # Load our location module
31 import location
.database
32 import location
.importer
33 from location
.i18n
import _
36 log
= logging
.getLogger("location.importer")
41 parser
= argparse
.ArgumentParser(
42 description
=_("Location Importer Command Line Interface"),
44 subparsers
= parser
.add_subparsers()
46 # Global configuration flags
47 parser
.add_argument("--debug", action
="store_true",
48 help=_("Enable debug output"))
49 parser
.add_argument("--quiet", action
="store_true",
50 help=_("Enable quiet mode"))
53 parser
.add_argument("--version", action
="version",
54 version
="%(prog)s @VERSION@")
57 parser
.add_argument("--database-host", required
=True,
58 help=_("Database Hostname"), metavar
=_("HOST"))
59 parser
.add_argument("--database-name", required
=True,
60 help=_("Database Name"), metavar
=_("NAME"))
61 parser
.add_argument("--database-username", required
=True,
62 help=_("Database Username"), metavar
=_("USERNAME"))
63 parser
.add_argument("--database-password", required
=True,
64 help=_("Database Password"), metavar
=_("PASSWORD"))
67 write
= subparsers
.add_parser("write", help=_("Write database to file"))
68 write
.set_defaults(func
=self
.handle_write
)
69 write
.add_argument("file", nargs
=1, help=_("Database File"))
70 write
.add_argument("--signing-key", nargs
="?", type=open, help=_("Signing Key"))
71 write
.add_argument("--backup-signing-key", nargs
="?", type=open, help=_("Backup Signing Key"))
72 write
.add_argument("--vendor", nargs
="?", help=_("Sets the vendor"))
73 write
.add_argument("--description", nargs
="?", help=_("Sets a description"))
74 write
.add_argument("--license", nargs
="?", help=_("Sets the license"))
75 write
.add_argument("--version", type=int, help=_("Database Format Version"))
78 update_whois
= subparsers
.add_parser("update-whois", help=_("Update WHOIS Information"))
79 update_whois
.set_defaults(func
=self
.handle_update_whois
)
81 # Update announcements
82 update_announcements
= subparsers
.add_parser("update-announcements",
83 help=_("Update BGP Annoucements"))
84 update_announcements
.set_defaults(func
=self
.handle_update_announcements
)
85 update_announcements
.add_argument("server", nargs
=1,
86 help=_("Route Server to connect to"), metavar
=_("SERVER"))
89 update_overrides
= subparsers
.add_parser("update-overrides",
90 help=_("Update overrides"),
92 update_overrides
.add_argument(
93 "files", nargs
="+", help=_("Files to import"),
95 update_overrides
.set_defaults(func
=self
.handle_update_overrides
)
98 import_countries
= subparsers
.add_parser("import-countries",
99 help=_("Import countries"),
101 import_countries
.add_argument("file", nargs
=1, type=argparse
.FileType("r"),
102 help=_("File to import"))
103 import_countries
.set_defaults(func
=self
.handle_import_countries
)
105 args
= parser
.parse_args()
109 location
.logger
.set_level(logging
.DEBUG
)
111 location
.logger
.set_level(logging
.WARNING
)
113 # Print usage if no action was given
114 if not "func" in args
:
121 # Parse command line arguments
122 args
= self
.parse_cli()
124 # Initialise database
125 self
.db
= self
._setup
_database
(args
)
128 ret
= args
.func(args
)
130 # Return with exit code
134 # Otherwise just exit
137 def _setup_database(self
, ns
):
139 Initialise the database
141 # Connect to database
142 db
= location
.database
.Connection(
143 host
=ns
.database_host
, database
=ns
.database_name
,
144 user
=ns
.database_username
, password
=ns
.database_password
,
147 with db
.transaction():
150 CREATE TABLE IF NOT EXISTS announcements(network inet, autnum bigint,
151 first_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP,
152 last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
153 CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
154 CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
155 CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops);
158 CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
159 CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number);
162 CREATE TABLE IF NOT EXISTS countries(
163 country_code text NOT NULL, name text NOT NULL, continent_code text NOT NULL);
164 CREATE UNIQUE INDEX IF NOT EXISTS countries_country_code ON countries(country_code);
167 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
168 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
169 CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
170 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
173 CREATE TABLE IF NOT EXISTS autnum_overrides(
174 number bigint NOT NULL,
177 is_anonymous_proxy boolean,
178 is_satellite_provider boolean,
181 CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number
182 ON autnum_overrides(number);
184 CREATE TABLE IF NOT EXISTS network_overrides(
185 network inet NOT NULL,
187 is_anonymous_proxy boolean,
188 is_satellite_provider boolean,
191 CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
192 ON network_overrides(network);
193 CREATE INDEX IF NOT EXISTS network_overrides_search
194 ON network_overrides USING GIST(network inet_ops);
199 def handle_write(self
, ns
):
201 Compiles a database in libloc format out of what is in the database
204 writer
= location
.Writer(ns
.signing_key
, ns
.backup_signing_key
)
208 writer
.vendor
= ns
.vendor
211 writer
.description
= ns
.description
214 writer
.license
= ns
.license
216 # Add all Autonomous Systems
217 log
.info("Writing Autonomous Systems...")
219 # Select all ASes with a name
220 rows
= self
.db
.query("""
222 autnums.number AS number,
224 (SELECT overrides.name FROM autnum_overrides overrides
225 WHERE overrides.number = autnums.number),
229 WHERE name <> %s ORDER BY number
233 a
= writer
.add_as(row
.number
)
237 log
.info("Writing networks...")
239 # Select all known networks
240 rows
= self
.db
.query("""
241 -- Return a list of those networks enriched with all
242 -- other information that we store in the database
244 DISTINCT ON (network)
251 SELECT country FROM network_overrides overrides
252 WHERE networks.network <<= overrides.network
253 ORDER BY masklen(overrides.network) DESC
257 SELECT country FROM autnum_overrides overrides
258 WHERE networks.autnum = overrides.number
266 SELECT is_anonymous_proxy FROM network_overrides overrides
267 WHERE networks.network <<= overrides.network
268 ORDER BY masklen(overrides.network) DESC
272 SELECT is_anonymous_proxy FROM autnum_overrides overrides
273 WHERE networks.autnum = overrides.number
276 ) AS is_anonymous_proxy,
279 SELECT is_satellite_provider FROM network_overrides overrides
280 WHERE networks.network <<= overrides.network
281 ORDER BY masklen(overrides.network) DESC
285 SELECT is_satellite_provider FROM autnum_overrides overrides
286 WHERE networks.autnum = overrides.number
289 ) AS is_satellite_provider,
292 SELECT is_anycast FROM network_overrides overrides
293 WHERE networks.network <<= overrides.network
294 ORDER BY masklen(overrides.network) DESC
298 SELECT is_anycast FROM autnum_overrides overrides
299 WHERE networks.autnum = overrides.number
305 known_networks.network AS network,
306 announcements.autnum AS autnum,
307 networks.country AS country,
309 -- Must be part of returned values for ORDER BY clause
310 masklen(announcements.network) AS sort_a,
311 masklen(networks.network) AS sort_b
313 SELECT network FROM announcements
315 SELECT network FROM networks
317 SELECT network FROM network_overrides
320 announcements ON known_networks.network <<= announcements.network
322 networks ON known_networks.network <<= networks.network
324 known_networks.network,
331 network
= writer
.add_network(row
.network
)
335 network
.country_code
= row
.country
339 network
.asn
= row
.autnum
342 if row
.is_anonymous_proxy
:
343 network
.set_flag(location
.NETWORK_FLAG_ANONYMOUS_PROXY
)
345 if row
.is_satellite_provider
:
346 network
.set_flag(location
.NETWORK_FLAG_SATELLITE_PROVIDER
)
349 network
.set_flag(location
.NETWORK_FLAG_ANYCAST
)
352 log
.info("Writing countries...")
353 rows
= self
.db
.query("SELECT * FROM countries ORDER BY country_code")
356 c
= writer
.add_country(row
.country_code
)
357 c
.continent_code
= row
.continent_code
360 # Write everything to file
361 log
.info("Writing database to file...")
365 def handle_update_whois(self
, ns
):
366 downloader
= location
.importer
.Downloader()
368 # Download all sources
369 with self
.db
.transaction():
370 # Create some temporary tables to store parsed data
372 CREATE TEMPORARY TABLE _autnums(number integer, organization text)
374 CREATE UNIQUE INDEX _autnums_number ON _autnums(number);
376 CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
378 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
380 CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
382 CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
383 CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
386 # Remove all previously imported content
388 TRUNCATE TABLE networks;
391 # Fetch all valid country codes to check parsed networks aganist...
392 rows
= self
.db
.query("SELECT * FROM countries ORDER BY country_code")
396 validcountries
.append(row
.country_code
)
398 for source
in location
.importer
.WHOIS_SOURCES
:
399 with downloader
.request(source
, return_blocks
=True) as f
:
401 self
._parse
_block
(block
, validcountries
)
403 # Process all parsed networks from every RIR we happen to have access to,
404 # insert the largest network chunks into the networks table immediately...
405 families
= self
.db
.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
407 for family
in (row
.family
for row
in families
):
408 smallest
= self
.db
.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family
)
410 self
.db
.execute("INSERT INTO networks(network, country) \
411 SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest
.prefix
, family
)
413 # ... determine any other prefixes for this network family, ...
414 prefixes
= self
.db
.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
415 WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family
)
417 # ... and insert networks with this prefix in case they provide additional
418 # information (i. e. subnet of a larger chunk with a different country)
419 for prefix
in (row
.prefix
for row
in prefixes
):
428 family(_rirdata.network) = %s
430 masklen(_rirdata.network) = %s
434 DISTINCT ON (c.network)
437 masklen(networks.network),
438 networks.country AS parent_country
444 c.network << networks.network
447 masklen(networks.network) DESC NULLS LAST
450 networks(network, country)
457 parent_country IS NULL
459 country <> parent_country
460 ON CONFLICT DO NOTHING""",
465 INSERT INTO autnums(number, name)
466 SELECT _autnums.number, _organizations.name FROM _autnums
467 JOIN _organizations ON _autnums.organization = _organizations.handle
468 ON CONFLICT (number) DO UPDATE SET name = excluded.name;
471 # Download all extended sources
472 for source
in location
.importer
.EXTENDED_SOURCES
:
473 with self
.db
.transaction():
475 with downloader
.request(source
) as f
:
477 self
._parse
_line
(line
, validcountries
)
479 def _check_parsed_network(self
, network
):
481 Assistive function to detect and subsequently sort out parsed
482 networks from RIR data (both Whois and so-called "extended sources"),
485 (a) not globally routable (RFC 1918 space, et al.)
486 (b) covering a too large chunk of the IP address space (prefix length
487 is < 7 for IPv4 networks, and < 10 for IPv6)
488 (c) "0.0.0.0" or "::" as a network address
489 (d) are too small for being publicly announced (we have decided not to
490 process them at the moment, as they significantly enlarge our
491 database without providing very helpful additional information)
493 This unfortunately is necessary due to brain-dead clutter across
494 various RIR databases, causing mismatches and eventually disruptions.
496 We will return False in case a network is not suitable for adding
497 it to our database, and True otherwise.
500 if not network
or not (isinstance(network
, ipaddress
.IPv4Network
) or isinstance(network
, ipaddress
.IPv6Network
)):
503 if not network
.is_global
:
504 log
.debug("Skipping non-globally routable network: %s" % network
)
507 if network
.version
== 4:
508 if network
.prefixlen
< 7:
509 log
.debug("Skipping too big IP chunk: %s" % network
)
512 if network
.prefixlen
> 24:
513 log
.debug("Skipping network too small to be publicly announced: %s" % network
)
516 if str(network
.network_address
) == "0.0.0.0":
517 log
.debug("Skipping network based on 0.0.0.0: %s" % network
)
520 elif network
.version
== 6:
521 if network
.prefixlen
< 10:
522 log
.debug("Skipping too big IP chunk: %s" % network
)
525 if network
.prefixlen
> 48:
526 log
.debug("Skipping network too small to be publicly announced: %s" % network
)
529 if str(network
.network_address
) == "::":
530 log
.debug("Skipping network based on '::': %s" % network
)
534 # This should not happen...
535 log
.warning("Skipping network of unknown family, this should not happen: %s" % network
)
538 # In case we have made it here, the network is considered to
539 # be suitable for libloc consumption...
542 def _parse_block(self
, block
, validcountries
= None):
543 # Get first line to find out what type of block this is
547 if line
.startswith("aut-num:"):
548 return self
._parse
_autnum
_block
(block
)
551 if line
.startswith("inet6num:") or line
.startswith("inetnum:"):
552 return self
._parse
_inetnum
_block
(block
, validcountries
)
555 elif line
.startswith("organisation:"):
556 return self
._parse
_org
_block
(block
)
558 def _parse_autnum_block(self
, block
):
562 key
, val
= split_line(line
)
565 m
= re
.match(r
"^(AS|as)(\d+)", val
)
567 autnum
["asn"] = m
.group(2)
570 autnum
[key
] = val
.upper()
576 # Insert into database
577 self
.db
.execute("INSERT INTO _autnums(number, organization) \
578 VALUES(%s, %s) ON CONFLICT (number) DO UPDATE SET \
579 organization = excluded.organization",
580 autnum
.get("asn"), autnum
.get("org"),
583 def _parse_inetnum_block(self
, block
, validcountries
= None):
584 log
.debug("Parsing inetnum block:")
591 key
, val
= split_line(line
)
593 # Filter any inetnum records which are only referring to IP space
594 # not managed by that specific RIR...
596 if re
.match(r
"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val
.strip()):
597 log
.debug("Skipping record indicating historic/orphaned data: %s" % val
.strip())
601 start_address
, delim
, end_address
= val
.partition("-")
603 # Strip any excess space
604 start_address
, end_address
= start_address
.rstrip(), end_address
.strip()
606 # Convert to IP address
608 start_address
= ipaddress
.ip_address(start_address
)
609 end_address
= ipaddress
.ip_address(end_address
)
611 log
.warning("Could not parse line: %s" % line
)
614 inetnum
["inetnum"] = list(ipaddress
.summarize_address_range(start_address
, end_address
))
616 elif key
== "inet6num":
617 inetnum
[key
] = [ipaddress
.ip_network(val
, strict
=False)]
619 elif key
== "country":
620 inetnum
[key
] = val
.upper()
623 if not inetnum
or not "country" in inetnum
:
626 # Iterate through all networks enumerated from above, check them for plausibility and insert
627 # them into the database, if _check_parsed_network() succeeded
628 for single_network
in inetnum
.get("inet6num") or inetnum
.get("inetnum"):
629 if self
._check
_parsed
_network
(single_network
):
631 # Skip objects with unknown country codes - to avoid log spam for invalid or too small
632 # networks, this check is - kinda ugly - done at this point
633 if validcountries
and inetnum
.get("country") not in validcountries
:
634 log
.warning("Skipping network with bogus country '%s': %s" % \
635 (inetnum
.get("country"), inetnum
.get("inet6num") or inetnum
.get("inetnum")))
638 # Everything is fine here, run INSERT statement...
639 self
.db
.execute("INSERT INTO _rirdata(network, country) \
640 VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
641 "%s" % single_network
, inetnum
.get("country"),
644 def _parse_org_block(self
, block
):
648 key
, val
= split_line(line
)
650 if key
== "organisation":
651 org
[key
] = val
.upper()
652 elif key
== "org-name":
659 self
.db
.execute("INSERT INTO _organizations(handle, name) \
660 VALUES(%s, %s) ON CONFLICT (handle) DO \
661 UPDATE SET name = excluded.name",
662 org
.get("organisation"), org
.get("org-name"),
665 def _parse_line(self
, line
, validcountries
= None):
667 if line
.startswith("2"):
671 if line
.startswith("#"):
675 registry
, country_code
, type, line
= line
.split("|", 3)
677 log
.warning("Could not parse line: %s" % line
)
680 # Skip any lines that are for stats only or do not have a country
681 # code at all (avoids log spam below)
682 if not country_code
or country_code
== '*':
685 # Skip objects with unknown country codes
686 if validcountries
and country_code
not in validcountries
:
687 log
.warning("Skipping line with bogus country '%s': %s" % \
688 (country_code
, line
))
691 if type in ("ipv6", "ipv4"):
692 return self
._parse
_ip
_line
(country_code
, type, line
)
694 def _parse_ip_line(self
, country
, type, line
):
696 address
, prefix
, date
, status
, organization
= line
.split("|")
700 # Try parsing the line without organization
702 address
, prefix
, date
, status
= line
.split("|")
704 log
.warning("Unhandled line format: %s" % line
)
707 # Skip anything that isn't properly assigned
708 if not status
in ("assigned", "allocated"):
711 # Cast prefix into an integer
715 log
.warning("Invalid prefix: %s" % prefix
)
718 # Fix prefix length for IPv4
720 prefix
= 32 - int(math
.log(prefix
, 2))
722 # Try to parse the address
724 network
= ipaddress
.ip_network("%s/%s" % (address
, prefix
), strict
=False)
726 log
.warning("Invalid IP address: %s" % address
)
729 if not self
._check
_parsed
_network
(network
):
732 self
.db
.execute("INSERT INTO networks(network, country) \
733 VALUES(%s, %s) ON CONFLICT (network) DO \
734 UPDATE SET country = excluded.country",
735 "%s" % network
, country
,
738 def handle_update_announcements(self
, ns
):
739 server
= ns
.server
[0]
741 with self
.db
.transaction():
742 if server
.startswith("/"):
743 self
._handle
_update
_announcements
_from
_bird
(server
)
745 self
._handle
_update
_announcements
_from
_telnet
(server
)
747 # Purge anything we never want here
749 -- Delete default routes
750 DELETE FROM announcements WHERE network = '::/0' OR network = '0.0.0.0/0';
752 -- Delete anything that is not global unicast address space
753 DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3';
755 -- DELETE "current network" address space
756 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '0.0.0.0/8';
758 -- DELETE local loopback address space
759 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '127.0.0.0/8';
761 -- DELETE RFC 1918 address space
762 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8';
763 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12';
764 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16';
766 -- DELETE test, benchmark and documentation address space
767 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.0.0/24';
768 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.2.0/24';
769 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.18.0.0/15';
770 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.51.100.0/24';
771 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '203.0.113.0/24';
773 -- DELETE CGNAT address space (RFC 6598)
774 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '100.64.0.0/10';
776 -- DELETE link local address space
777 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '169.254.0.0/16';
779 -- DELETE IPv6 to IPv4 (6to4) address space (RFC 3068)
780 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.88.99.0/24';
781 DELETE FROM announcements WHERE family(network) = 6 AND network <<= '2002::/16';
783 -- DELETE multicast and reserved address space
784 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '224.0.0.0/4';
785 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '240.0.0.0/4';
787 -- Delete networks that are too small to be in the global routing table
788 DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48;
789 DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24;
791 -- Delete any non-public or reserved ASNs
792 DELETE FROM announcements WHERE NOT (
793 (autnum >= 1 AND autnum <= 23455)
795 (autnum >= 23457 AND autnum <= 64495)
797 (autnum >= 131072 AND autnum <= 4199999999)
800 -- Delete everything that we have not seen for 14 days
801 DELETE FROM announcements WHERE last_seen_at <= CURRENT_TIMESTAMP - INTERVAL '14 days';
804 def _handle_update_announcements_from_bird(self
, server
):
805 # Pre-compile the regular expression for faster searching
806 route
= re
.compile(b
"^\s(.+?)\s+.+?\[AS(.*?).\]$")
808 log
.info("Requesting routing table from Bird (%s)" % server
)
810 # Send command to list all routes
811 for line
in self
._bird
_cmd
(server
, "show route"):
812 m
= route
.match(line
)
814 log
.debug("Could not parse line: %s" % line
.decode())
817 # Fetch the extracted network and ASN
818 network
, autnum
= m
.groups()
820 # Insert it into the database
821 self
.db
.execute("INSERT INTO announcements(network, autnum) \
822 VALUES(%s, %s) ON CONFLICT (network) DO \
823 UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
824 network
.decode(), autnum
.decode(),
827 def _handle_update_announcements_from_telnet(self
, server
):
828 # Pre-compile regular expression for routes
829 route
= re
.compile(b
"^\*[\s\>]i([^\s]+).+?(\d+)\si\r\n", re
.MULTILINE|re
.DOTALL
)
831 with telnetlib
.Telnet(server
) as t
:
834 # t.set_debuglevel(10)
836 # Wait for console greeting
837 greeting
= t
.read_until(b
"> ", timeout
=30)
839 log
.error("Could not get a console prompt")
843 t
.write(b
"terminal length 0\n")
845 # Wait for the prompt to return
848 # Fetch the routing tables
849 for protocol
in ("ipv6", "ipv4"):
850 log
.info("Requesting %s routing table" % protocol
)
852 # Request the full unicast routing table
853 t
.write(b
"show bgp %s unicast\n" % protocol
.encode())
855 # Read entire header which ends with "Path"
856 t
.read_until(b
"Path\r\n")
859 # Try reading a full entry
860 # Those might be broken across multiple lines but ends with i
861 line
= t
.read_until(b
"i\r\n", timeout
=5)
865 # Show line for debugging
866 #log.debug(repr(line))
868 # Try finding a route in here
869 m
= route
.match(line
)
871 network
, autnum
= m
.groups()
873 # Convert network to string
874 network
= network
.decode()
876 # Append /24 for IPv4 addresses
877 if not "/" in network
and not ":" in network
:
878 network
= "%s/24" % network
880 # Convert AS number to integer
883 log
.info("Found announcement for %s by %s" % (network
, autnum
))
885 self
.db
.execute("INSERT INTO announcements(network, autnum) \
886 VALUES(%s, %s) ON CONFLICT (network) DO \
887 UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
891 log
.info("Finished reading the %s routing table" % protocol
)
893 def _bird_cmd(self
, socket_path
, command
):
894 # Connect to the socket
895 s
= socket
.socket(socket
.AF_UNIX
, socket
.SOCK_STREAM
)
896 s
.connect(socket_path
)
898 # Allocate some buffer
902 s
.send(b
"%s\n" % command
.encode())
906 buffer += s
.recv(4096)
909 # Search for the next newline
910 pos
= buffer.find(b
"\n")
912 # If we cannot find one, we go back and read more data
916 # Cut after the newline character
919 # Split the line we want and keep the rest in buffer
920 line
, buffer = buffer[:pos
], buffer[pos
:]
922 # Look for the end-of-output indicator
923 if line
== b
"0000 \n":
926 # Otherwise return the line
929 def handle_update_overrides(self
, ns
):
930 with self
.db
.transaction():
931 # Drop all data that we have
933 TRUNCATE TABLE autnum_overrides;
934 TRUNCATE TABLE network_overrides;
937 for file in ns
.files
:
938 log
.info("Reading %s..." % file)
940 with
open(file, "rb") as f
:
941 for type, block
in location
.importer
.read_blocks(f
):
943 network
= block
.get("net")
944 # Try to parse and normalise the network
946 network
= ipaddress
.ip_network(network
, strict
=False)
947 except ValueError as e
:
948 log
.warning("Invalid IP network: %s: %s" % (network
, e
))
951 # Prevent that we overwrite all networks
952 if network
.prefixlen
== 0:
953 log
.warning("Skipping %s: You cannot overwrite default" % network
)
957 INSERT INTO network_overrides(
961 is_satellite_provider,
963 ) VALUES (%s, %s, %s, %s, %s)
964 ON CONFLICT (network) DO NOTHING""",
966 block
.get("country"),
967 self
._parse
_bool
(block
, "is-anonymous-proxy"),
968 self
._parse
_bool
(block
, "is-satellite-provider"),
969 self
._parse
_bool
(block
, "is-anycast"),
972 elif type == "aut-num":
973 autnum
= block
.get("aut-num")
975 # Check if AS number begins with "AS"
976 if not autnum
.startswith("AS"):
977 log
.warning("Invalid AS number: %s" % autnum
)
984 INSERT INTO autnum_overrides(
989 is_satellite_provider,
991 ) VALUES(%s, %s, %s, %s, %s, %s)
992 ON CONFLICT DO NOTHING""",
995 block
.get("country"),
996 self
._parse
_bool
(block
, "is-anonymous-proxy"),
997 self
._parse
_bool
(block
, "is-satellite-provider"),
998 self
._parse
_bool
(block
, "is-anycast"),
1002 log
.warning("Unsupported type: %s" % type)
1005 def _parse_bool(block
, key
):
1006 val
= block
.get(key
)
1008 # There is no point to proceed when we got None
1012 # Convert to lowercase
1016 if val
in ("yes", "1"):
1020 if val
in ("no", "0"):
1026 def handle_import_countries(self
, ns
):
1027 with self
.db
.transaction():
1028 # Drop all data that we have
1029 self
.db
.execute("TRUNCATE TABLE countries")
1031 for file in ns
.file:
1033 line
= line
.rstrip()
1035 # Ignore any comments
1036 if line
.startswith("#"):
1040 country_code
, continent_code
, name
= line
.split(maxsplit
=2)
1042 log
.warning("Could not parse line: %s" % line
)
1045 self
.db
.execute("INSERT INTO countries(country_code, name, continent_code) \
1046 VALUES(%s, %s, %s) ON CONFLICT DO NOTHING", country_code
, name
, continent_code
)
1049 def split_line(line
):
1050 key
, colon
, val
= line
.partition(":")
1052 # Strip any excess space
1059 # Run the command line interface