2 ###############################################################################
4 # libloc - A library to determine the location of someone on the Internet #
6 # Copyright (C) 2020-2024 IPFire Development Team <info@ipfire.org> #
8 # This library is free software; you can redistribute it and/or #
9 # modify it under the terms of the GNU Lesser General Public #
10 # License as published by the Free Software Foundation; either #
11 # version 2.1 of the License, or (at your option) any later version. #
13 # This library is distributed in the hope that it will be useful, #
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
16 # Lesser General Public License for more details. #
18 ###############################################################################
21 import concurrent
.futures
34 # Load our location module
36 import location
.database
37 import location
.importer
38 from location
.i18n
import _
41 log
= logging
.getLogger("location.importer")
51 TRANSLATED_COUNTRIES
= {
52 # When people say UK, they mean GB
56 IGNORED_COUNTRIES
= set((
60 # Some people use ZZ to say "no country" or to hide the country
64 # Configure the CSV parser for ARIN
65 csv
.register_dialect("arin", delimiter
=",", quoting
=csv
.QUOTE_ALL
, quotechar
="\"")
69 parser
= argparse
.ArgumentParser(
70 description
=_("Location Importer Command Line Interface"),
72 subparsers
= parser
.add_subparsers()
74 # Global configuration flags
75 parser
.add_argument("--debug", action
="store_true",
76 help=_("Enable debug output"))
77 parser
.add_argument("--quiet", action
="store_true",
78 help=_("Enable quiet mode"))
81 parser
.add_argument("--version", action
="version",
82 version
="%(prog)s @VERSION@")
85 parser
.add_argument("--database-host", required
=True,
86 help=_("Database Hostname"), metavar
=_("HOST"))
87 parser
.add_argument("--database-name", required
=True,
88 help=_("Database Name"), metavar
=_("NAME"))
89 parser
.add_argument("--database-username", required
=True,
90 help=_("Database Username"), metavar
=_("USERNAME"))
91 parser
.add_argument("--database-password", required
=True,
92 help=_("Database Password"), metavar
=_("PASSWORD"))
95 write
= subparsers
.add_parser("write", help=_("Write database to file"))
96 write
.set_defaults(func
=self
.handle_write
)
97 write
.add_argument("file", nargs
=1, help=_("Database File"))
98 write
.add_argument("--signing-key", nargs
="?", type=open, help=_("Signing Key"))
99 write
.add_argument("--backup-signing-key", nargs
="?", type=open, help=_("Backup Signing Key"))
100 write
.add_argument("--vendor", nargs
="?", help=_("Sets the vendor"))
101 write
.add_argument("--description", nargs
="?", help=_("Sets a description"))
102 write
.add_argument("--license", nargs
="?", help=_("Sets the license"))
103 write
.add_argument("--version", type=int, help=_("Database Format Version"))
106 update_whois
= subparsers
.add_parser("update-whois", help=_("Update WHOIS Information"))
107 update_whois
.set_defaults(func
=self
.handle_update_whois
)
109 # Update announcements
110 update_announcements
= subparsers
.add_parser("update-announcements",
111 help=_("Update BGP Annoucements"))
112 update_announcements
.set_defaults(func
=self
.handle_update_announcements
)
113 update_announcements
.add_argument("server", nargs
=1,
114 help=_("Route Server to connect to"), metavar
=_("SERVER"))
117 update_geofeeds
= subparsers
.add_parser("update-geofeeds",
118 help=_("Update Geofeeds"))
119 update_geofeeds
.set_defaults(func
=self
.handle_update_geofeeds
)
122 update_feeds
= subparsers
.add_parser("update-feeds",
123 help=_("Update Feeds"))
124 update_feeds
.add_argument("feeds", nargs
="*",
125 help=_("Only update these feeds"))
126 update_feeds
.set_defaults(func
=self
.handle_update_feeds
)
129 update_overrides
= subparsers
.add_parser("update-overrides",
130 help=_("Update overrides"),
132 update_overrides
.add_argument(
133 "files", nargs
="+", help=_("Files to import"),
135 update_overrides
.set_defaults(func
=self
.handle_update_overrides
)
138 import_countries
= subparsers
.add_parser("import-countries",
139 help=_("Import countries"),
141 import_countries
.add_argument("file", nargs
=1, type=argparse
.FileType("r"),
142 help=_("File to import"))
143 import_countries
.set_defaults(func
=self
.handle_import_countries
)
145 args
= parser
.parse_args()
149 location
.logger
.set_level(logging
.DEBUG
)
151 location
.logger
.set_level(logging
.WARNING
)
153 # Print usage if no action was given
154 if not "func" in args
:
161 # Parse command line arguments
162 args
= self
.parse_cli()
164 # Initialise database
165 self
.db
= self
._setup
_database
(args
)
168 ret
= args
.func(args
)
170 # Return with exit code
174 # Otherwise just exit
177 def _setup_database(self
, ns
):
179 Initialise the database
181 # Connect to database
182 db
= location
.database
.Connection(
183 host
=ns
.database_host
, database
=ns
.database_name
,
184 user
=ns
.database_username
, password
=ns
.database_password
,
187 with db
.transaction():
190 CREATE TABLE IF NOT EXISTS announcements(network inet, autnum bigint,
191 first_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP,
192 last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
193 CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
194 CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
195 CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops);
198 CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
199 ALTER TABLE autnums ADD COLUMN IF NOT EXISTS source text;
200 CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number);
203 CREATE TABLE IF NOT EXISTS countries(
204 country_code text NOT NULL, name text NOT NULL, continent_code text NOT NULL);
205 CREATE UNIQUE INDEX IF NOT EXISTS countries_country_code ON countries(country_code);
208 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
209 ALTER TABLE networks ADD COLUMN IF NOT EXISTS original_countries text[];
210 ALTER TABLE networks ADD COLUMN IF NOT EXISTS source text;
211 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
212 CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
213 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
216 CREATE TABLE IF NOT EXISTS geofeeds(
217 id serial primary key,
219 status integer default null,
220 updated_at timestamp without time zone default null
222 ALTER TABLE geofeeds ADD COLUMN IF NOT EXISTS error text;
223 CREATE UNIQUE INDEX IF NOT EXISTS geofeeds_unique
225 CREATE TABLE IF NOT EXISTS geofeed_networks(
226 geofeed_id integer references geofeeds(id) on delete cascade,
232 CREATE INDEX IF NOT EXISTS geofeed_networks_geofeed_id
233 ON geofeed_networks(geofeed_id);
234 CREATE INDEX IF NOT EXISTS geofeed_networks_search
235 ON geofeed_networks USING GIST(network inet_ops);
236 CREATE TABLE IF NOT EXISTS network_geofeeds(network inet, url text);
237 CREATE UNIQUE INDEX IF NOT EXISTS network_geofeeds_unique
238 ON network_geofeeds(network);
239 CREATE INDEX IF NOT EXISTS network_geofeeds_search
240 ON network_geofeeds USING GIST(network inet_ops);
241 CREATE INDEX IF NOT EXISTS network_geofeeds_url
242 ON network_geofeeds(url);
245 CREATE TABLE IF NOT EXISTS autnum_feeds(
246 number bigint NOT NULL,
247 source text NOT NULL,
250 is_anonymous_proxy boolean,
251 is_satellite_provider boolean,
255 CREATE UNIQUE INDEX IF NOT EXISTS autnum_feeds_unique
256 ON autnum_feeds(number, source);
258 CREATE TABLE IF NOT EXISTS network_feeds(
259 network inet NOT NULL,
260 source text NOT NULL,
262 is_anonymous_proxy boolean,
263 is_satellite_provider boolean,
267 CREATE UNIQUE INDEX IF NOT EXISTS network_feeds_unique
268 ON network_feeds(network, source);
269 CREATE INDEX IF NOT EXISTS network_feeds_search
270 ON network_feeds USING GIST(network inet_ops);
273 CREATE TABLE IF NOT EXISTS autnum_overrides(
274 number bigint NOT NULL,
277 is_anonymous_proxy boolean,
278 is_satellite_provider boolean,
281 CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number
282 ON autnum_overrides(number);
283 ALTER TABLE autnum_overrides ADD COLUMN IF NOT EXISTS is_drop boolean;
284 ALTER TABLE autnum_overrides DROP COLUMN IF EXISTS source;
286 CREATE TABLE IF NOT EXISTS network_overrides(
287 network inet NOT NULL,
289 is_anonymous_proxy boolean,
290 is_satellite_provider boolean,
293 CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
294 ON network_overrides(network);
295 CREATE INDEX IF NOT EXISTS network_overrides_search
296 ON network_overrides USING GIST(network inet_ops);
297 ALTER TABLE network_overrides ADD COLUMN IF NOT EXISTS is_drop boolean;
298 ALTER TABLE network_overrides DROP COLUMN IF EXISTS source;
303 def fetch_countries(self
):
305 Returns a list of all countries on the list
307 # Fetch all valid country codes to check parsed networks aganist...
308 countries
= self
.db
.query("SELECT country_code FROM countries ORDER BY country_code")
310 return [country
.country_code
for country
in countries
]
312 def handle_write(self
, ns
):
314 Compiles a database in libloc format out of what is in the database
317 writer
= location
.Writer(ns
.signing_key
, ns
.backup_signing_key
)
321 writer
.vendor
= ns
.vendor
324 writer
.description
= ns
.description
327 writer
.license
= ns
.license
329 # Add all Autonomous Systems
330 log
.info("Writing Autonomous Systems...")
332 # Select all ASes with a name
333 rows
= self
.db
.query("""
335 autnums.number AS number,
343 autnum_overrides overrides ON autnums.number = overrides.number
349 # Skip AS without names
353 a
= writer
.add_as(row
.number
)
357 log
.info("Writing networks...")
359 # Select all known networks
360 rows
= self
.db
.query("""
361 WITH known_networks AS (
362 SELECT network FROM announcements
364 SELECT network FROM networks
366 SELECT network FROM network_feeds
368 SELECT network FROM network_overrides
370 SELECT network FROM geofeed_networks
373 ordered_networks AS (
375 known_networks.network AS network,
376 announcements.autnum AS autnum,
377 networks.country AS country,
379 -- Must be part of returned values for ORDER BY clause
380 masklen(announcements.network) AS sort_a,
381 masklen(networks.network) AS sort_b
385 announcements ON known_networks.network <<= announcements.network
387 networks ON known_networks.network <<= networks.network
389 known_networks.network,
394 -- Return a list of those networks enriched with all
395 -- other information that we store in the database
397 DISTINCT ON (network)
407 network_overrides overrides
409 networks.network <<= overrides.network
411 masklen(overrides.network) DESC
418 autnum_overrides overrides
420 networks.autnum = overrides.number
428 networks.network <<= feeds.network
430 masklen(feeds.network) DESC
439 networks.autnum = feeds.number
446 geofeed_networks.country AS country
450 -- Join the data from the geofeeds
452 geofeeds ON network_geofeeds.url = geofeeds.url
454 geofeed_networks ON geofeeds.id = geofeed_networks.geofeed_id
456 -- Check whether we have a geofeed for this network
458 networks.network <<= network_geofeeds.network
460 networks.network <<= geofeed_networks.network
462 -- Filter for the best result
464 masklen(geofeed_networks.network) DESC
476 network_overrides overrides
478 networks.network <<= overrides.network
480 masklen(overrides.network) DESC
489 networks.network <<= feeds.network
491 masklen(feeds.network) DESC
500 networks.autnum = feeds.number
509 autnum_overrides overrides
511 networks.autnum = overrides.number
514 ) AS is_anonymous_proxy,
518 is_satellite_provider
520 network_overrides overrides
522 networks.network <<= overrides.network
524 masklen(overrides.network) DESC
529 is_satellite_provider
533 networks.network <<= feeds.network
535 masklen(feeds.network) DESC
540 is_satellite_provider
544 networks.autnum = feeds.number
551 is_satellite_provider
553 autnum_overrides overrides
555 networks.autnum = overrides.number
558 ) AS is_satellite_provider,
564 network_overrides overrides
566 networks.network <<= overrides.network
568 masklen(overrides.network) DESC
577 networks.network <<= feeds.network
579 masklen(feeds.network) DESC
588 networks.autnum = feeds.number
597 autnum_overrides overrides
599 networks.autnum = overrides.number
608 network_overrides overrides
610 networks.network <<= overrides.network
612 masklen(overrides.network) DESC
621 networks.network <<= feeds.network
623 masklen(feeds.network) DESC
632 networks.autnum = feeds.number
641 autnum_overrides overrides
643 networks.autnum = overrides.number
648 ordered_networks networks
652 network
= writer
.add_network(row
.network
)
656 network
.country_code
= row
.country
660 network
.asn
= row
.autnum
663 if row
.is_anonymous_proxy
:
664 network
.set_flag(location
.NETWORK_FLAG_ANONYMOUS_PROXY
)
666 if row
.is_satellite_provider
:
667 network
.set_flag(location
.NETWORK_FLAG_SATELLITE_PROVIDER
)
670 network
.set_flag(location
.NETWORK_FLAG_ANYCAST
)
673 network
.set_flag(location
.NETWORK_FLAG_DROP
)
676 log
.info("Writing countries...")
677 rows
= self
.db
.query("SELECT * FROM countries ORDER BY country_code")
680 c
= writer
.add_country(row
.country_code
)
681 c
.continent_code
= row
.continent_code
684 # Write everything to file
685 log
.info("Writing database to file...")
689 def handle_update_whois(self
, ns
):
690 downloader
= location
.importer
.Downloader()
692 # Did we run successfully?
695 # Fetch all valid country codes to check parsed networks against
696 countries
= self
.fetch_countries()
698 # Check if we have countries
700 log
.error("Please import countries before importing any WHOIS data")
703 # Iterate over all potential sources
704 for source
in sorted(location
.importer
.SOURCES
):
705 with self
.db
.transaction():
706 # Create some temporary tables to store parsed data
708 CREATE TEMPORARY TABLE _autnums(number integer NOT NULL,
709 organization text NOT NULL, source text NOT NULL) ON COMMIT DROP;
710 CREATE UNIQUE INDEX _autnums_number ON _autnums(number);
712 CREATE TEMPORARY TABLE _organizations(handle text NOT NULL,
713 name text NOT NULL, source text NOT NULL) ON COMMIT DROP;
714 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
716 CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL,
717 original_countries text[] NOT NULL, source text NOT NULL)
719 CREATE INDEX _rirdata_search ON _rirdata
720 USING BTREE(family(network), masklen(network));
721 CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
724 # Remove all previously imported content
725 self
.db
.execute("DELETE FROM autnums WHERE source = %s", source
)
726 self
.db
.execute("DELETE FROM networks WHERE source = %s", source
)
729 # Fetch WHOIS sources
730 for url
in location
.importer
.WHOIS_SOURCES
.get(source
, []):
731 for block
in downloader
.request_blocks(url
):
732 self
._parse
_block
(block
, source
, countries
)
734 # Fetch extended sources
735 for url
in location
.importer
.EXTENDED_SOURCES
.get(source
, []):
736 for line
in downloader
.request_lines(url
):
737 self
._parse
_line
(line
, source
, countries
)
738 except urllib
.error
.URLError
as e
:
739 log
.error("Could not retrieve data from %s: %s" % (source
, e
))
742 # Continue with the next source
745 # Process all parsed networks from every RIR we happen to have access to,
746 # insert the largest network chunks into the networks table immediately...
747 families
= self
.db
.query("""
749 family(network) AS family
757 for family
in (row
.family
for row
in families
):
758 # Fetch the smallest mask length in our data set
759 smallest
= self
.db
.get("""
789 masklen(network) = %s
798 # ... determine any other prefixes for this network family, ...
799 prefixes
= self
.db
.query("""
801 DISTINCT masklen(network) AS prefix
812 # ... and insert networks with this prefix in case they provide additional
813 # information (i. e. subnet of a larger chunk with a different country)
814 for prefix
in (row
.prefix
for row
in prefixes
):
820 _rirdata.original_countries,
825 family(_rirdata.network) = %s
827 masklen(_rirdata.network) = %s
831 DISTINCT ON (c.network)
834 c.original_countries,
836 masklen(networks.network),
837 networks.country AS parent_country
843 c.network << networks.network
846 masklen(networks.network) DESC NULLS LAST
849 networks(network, country, original_countries, source)
858 parent_country IS NULL
860 country <> parent_country
861 ON CONFLICT DO NOTHING
876 _organizations.source
880 _organizations ON _autnums.organization = _organizations.handle
886 SET name = excluded.name
890 # Download and import (technical) AS names from ARIN
891 with self
.db
.transaction():
892 self
._import
_as
_names
_from
_arin
(downloader
)
894 # Return a non-zero exit code for errors
895 return 1 if error
else 0
897 def _check_parsed_network(self
, network
):
899 Assistive function to detect and subsequently sort out parsed
900 networks from RIR data (both Whois and so-called "extended sources"),
903 (a) not globally routable (RFC 1918 space, et al.)
904 (b) covering a too large chunk of the IP address space (prefix length
905 is < 7 for IPv4 networks, and < 10 for IPv6)
906 (c) "0.0.0.0" or "::" as a network address
907 (d) are too small for being publicly announced (we have decided not to
908 process them at the moment, as they significantly enlarge our
909 database without providing very helpful additional information)
911 This unfortunately is necessary due to brain-dead clutter across
912 various RIR databases, causing mismatches and eventually disruptions.
914 We will return False in case a network is not suitable for adding
915 it to our database, and True otherwise.
918 if not network
or not (isinstance(network
, ipaddress
.IPv4Network
) or isinstance(network
, ipaddress
.IPv6Network
)):
921 if not network
.is_global
:
922 log
.debug("Skipping non-globally routable network: %s" % network
)
925 if network
.version
== 4:
926 if network
.prefixlen
< 7:
927 log
.debug("Skipping too big IP chunk: %s" % network
)
930 if network
.prefixlen
> 24:
931 log
.debug("Skipping network too small to be publicly announced: %s" % network
)
934 if str(network
.network_address
) == "0.0.0.0":
935 log
.debug("Skipping network based on 0.0.0.0: %s" % network
)
938 elif network
.version
== 6:
939 if network
.prefixlen
< 10:
940 log
.debug("Skipping too big IP chunk: %s" % network
)
943 if network
.prefixlen
> 48:
944 log
.debug("Skipping network too small to be publicly announced: %s" % network
)
947 if str(network
.network_address
) == "::":
948 log
.debug("Skipping network based on '::': %s" % network
)
952 # This should not happen...
953 log
.warning("Skipping network of unknown family, this should not happen: %s" % network
)
956 # In case we have made it here, the network is considered to
957 # be suitable for libloc consumption...
960 def _check_parsed_asn(self
, asn
):
962 Assistive function to filter Autonomous System Numbers not being suitable
963 for adding to our database. Returns False in such cases, and True otherwise.
966 for start
, end
in VALID_ASN_RANGES
:
967 if start
<= asn
and end
>= asn
:
970 log
.info("Supplied ASN %s out of publicly routable ASN ranges" % asn
)
973 def _parse_block(self
, block
, source_key
, countries
):
974 # Get first line to find out what type of block this is
978 if line
.startswith("aut-num:"):
979 return self
._parse
_autnum
_block
(block
, source_key
)
982 if line
.startswith("inet6num:") or line
.startswith("inetnum:"):
983 return self
._parse
_inetnum
_block
(block
, source_key
, countries
)
986 elif line
.startswith("organisation:"):
987 return self
._parse
_org
_block
(block
, source_key
)
989 def _parse_autnum_block(self
, block
, source_key
):
993 key
, val
= split_line(line
)
996 m
= re
.match(r
"^(AS|as)(\d+)", val
)
998 autnum
["asn"] = m
.group(2)
1001 autnum
[key
] = val
.upper()
1003 elif key
== "descr":
1004 # Save the first description line as well...
1005 if not key
in autnum
:
1008 # Skip empty objects
1009 if not autnum
or not "asn" in autnum
:
1012 # Insert a dummy organisation handle into our temporary organisations
1013 # table in case the AS does not have an organisation handle set, but
1014 # has a description (a quirk often observed in APNIC area), so we can
1015 # later display at least some string for this AS.
1016 if not "org" in autnum
:
1017 if "descr" in autnum
:
1018 autnum
["org"] = "LIBLOC-%s-ORGHANDLE" % autnum
.get("asn")
1020 self
.db
.execute("INSERT INTO _organizations(handle, name, source) \
1021 VALUES(%s, %s, %s) ON CONFLICT (handle) DO NOTHING",
1022 autnum
.get("org"), autnum
.get("descr"), source_key
,
1025 log
.warning("ASN %s neither has an organisation handle nor a description line set, omitting" % \
1029 # Insert into database
1030 self
.db
.execute("INSERT INTO _autnums(number, organization, source) \
1031 VALUES(%s, %s, %s) ON CONFLICT (number) DO UPDATE SET \
1032 organization = excluded.organization",
1033 autnum
.get("asn"), autnum
.get("org"), source_key
,
1036 def _parse_inetnum_block(self
, block
, source_key
, countries
):
1037 log
.debug("Parsing inetnum block:")
1044 key
, val
= split_line(line
)
1046 # Filter any inetnum records which are only referring to IP space
1047 # not managed by that specific RIR...
1048 if key
== "netname":
1049 if re
.match(r
"^(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK|STUB-[\d-]{3,}SLASH\d{1,2})", val
.strip()):
1050 log
.debug("Skipping record indicating historic/orphaned data: %s" % val
.strip())
1053 if key
== "inetnum":
1054 start_address
, delim
, end_address
= val
.partition("-")
1056 # Strip any excess space
1057 start_address
, end_address
= start_address
.rstrip(), end_address
.strip()
1059 # Handle "inetnum" formatting in LACNIC DB (e.g. "24.152.8/22" instead of "24.152.8.0/22")
1060 if start_address
and not (delim
or end_address
):
1062 start_address
= ipaddress
.ip_network(start_address
, strict
=False)
1064 start_address
= start_address
.split("/")
1065 ldigits
= start_address
[0].count(".")
1067 # How many octets do we need to add?
1068 # (LACNIC does not seem to have a /8 or greater assigned, so the following should suffice.)
1070 start_address
= start_address
[0] + ".0.0/" + start_address
[1]
1072 start_address
= start_address
[0] + ".0/" + start_address
[1]
1074 log
.warning("Could not recover IPv4 address from line in LACNIC DB format: %s" % line
)
1078 start_address
= ipaddress
.ip_network(start_address
, strict
=False)
1080 log
.warning("Could not parse line in LACNIC DB format: %s" % line
)
1083 # Enumerate first and last IP address of this network
1084 end_address
= start_address
[-1]
1085 start_address
= start_address
[0]
1088 # Convert to IP address
1090 start_address
= ipaddress
.ip_address(start_address
)
1091 end_address
= ipaddress
.ip_address(end_address
)
1093 log
.warning("Could not parse line: %s" % line
)
1096 inetnum
["inetnum"] = list(ipaddress
.summarize_address_range(start_address
, end_address
))
1098 elif key
== "inet6num":
1099 inetnum
[key
] = [ipaddress
.ip_network(val
, strict
=False)]
1101 elif key
== "country":
1104 # Catch RIR data objects with more than one country code...
1105 if not key
in inetnum
:
1108 if val
in inetnum
.get("country"):
1109 # ... but keep this list distinct...
1112 # Ignore certain country codes
1113 if val
in IGNORED_COUNTRIES
:
1114 log
.debug("Ignoring country code '%s'" % val
)
1117 # Translate country codes
1119 val
= TRANSLATED_COUNTRIES
[val
]
1123 inetnum
[key
].append(val
)
1125 # Parse the geofeed attribute
1126 elif key
== "geofeed":
1127 inetnum
["geofeed"] = val
1129 # Parse geofeed when used as a remark
1130 elif key
== "remarks":
1131 m
= re
.match(r
"^(?:Geofeed)\s+(https://.*)", val
)
1133 inetnum
["geofeed"] = m
.group(1)
1135 # Skip empty objects
1136 if not inetnum
or not "country" in inetnum
:
1139 # Prepare skipping objects with unknown country codes...
1140 invalidcountries
= [singlecountry
for singlecountry
in inetnum
.get("country") if singlecountry
not in countries
]
1142 # Iterate through all networks enumerated from above, check them for plausibility and insert
1143 # them into the database, if _check_parsed_network() succeeded
1144 for single_network
in inetnum
.get("inet6num") or inetnum
.get("inetnum"):
1145 if self
._check
_parsed
_network
(single_network
):
1146 # Skip objects with unknown country codes if they are valid to avoid log spam...
1147 if invalidcountries
:
1148 log
.warning("Skipping network with bogus countr(y|ies) %s (original countries: %s): %s" % \
1149 (invalidcountries
, inetnum
.get("country"), inetnum
.get("inet6num") or inetnum
.get("inetnum")))
1152 # Everything is fine here, run INSERT statement...
1153 self
.db
.execute("INSERT INTO _rirdata(network, country, original_countries, source) \
1154 VALUES(%s, %s, %s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
1155 "%s" % single_network
, inetnum
.get("country")[0], inetnum
.get("country"), source_key
,
1158 # Update any geofeed information
1159 geofeed
= inetnum
.get("geofeed", None)
1161 self
._parse
_geofeed
(geofeed
, single_network
)
1163 # Delete any previous geofeeds
1165 self
.db
.execute("DELETE FROM network_geofeeds WHERE network = %s",
1166 "%s" % single_network
)
1168 def _parse_geofeed(self
, url
, single_network
):
1170 url
= urllib
.parse
.urlparse(url
)
1172 # Make sure that this is a HTTPS URL
1173 if not url
.scheme
== "https":
1174 log
.debug("Geofeed URL is not using HTTPS: %s" % geofeed
)
1177 # Put the URL back together normalized
1180 # Store/update any geofeeds
1190 ON CONFLICT (network) DO
1191 UPDATE SET url = excluded.url""",
1192 "%s" % single_network
, url
,
1195 def _parse_org_block(self
, block
, source_key
):
1199 key
, val
= split_line(line
)
1201 if key
== "organisation":
1202 org
[key
] = val
.upper()
1203 elif key
== "org-name":
1206 # Skip empty objects
1210 self
.db
.execute("INSERT INTO _organizations(handle, name, source) \
1211 VALUES(%s, %s, %s) ON CONFLICT (handle) DO \
1212 UPDATE SET name = excluded.name",
1213 org
.get("organisation"), org
.get("org-name"), source_key
,
1216 def _parse_line(self
, line
, source_key
, validcountries
=None):
1218 if line
.startswith("2"):
1222 if line
.startswith("#"):
1226 registry
, country_code
, type, line
= line
.split("|", 3)
1228 log
.warning("Could not parse line: %s" % line
)
1231 # Skip any unknown protocols
1232 if not type in ("ipv6", "ipv4"):
1233 log
.warning("Unknown IP protocol '%s'" % type)
1236 # Skip any lines that are for stats only or do not have a country
1237 # code at all (avoids log spam below)
1238 if not country_code
or country_code
== '*':
1241 # Skip objects with unknown country codes
1242 if validcountries
and country_code
not in validcountries
:
1243 log
.warning("Skipping line with bogus country '%s': %s" % \
1244 (country_code
, line
))
1248 address
, prefix
, date
, status
, organization
= line
.split("|")
1252 # Try parsing the line without organization
1254 address
, prefix
, date
, status
= line
.split("|")
1256 log
.warning("Unhandled line format: %s" % line
)
1259 # Skip anything that isn't properly assigned
1260 if not status
in ("assigned", "allocated"):
1263 # Cast prefix into an integer
1265 prefix
= int(prefix
)
1267 log
.warning("Invalid prefix: %s" % prefix
)
1270 # Fix prefix length for IPv4
1272 prefix
= 32 - int(math
.log(prefix
, 2))
1274 # Try to parse the address
1276 network
= ipaddress
.ip_network("%s/%s" % (address
, prefix
), strict
=False)
1278 log
.warning("Invalid IP address: %s" % address
)
1281 if not self
._check
_parsed
_network
(network
):
1297 ON CONFLICT (network)
1298 DO UPDATE SET country = excluded.country
1299 """, "%s" % network
, country_code
, [country
], source_key
,
1302 def _import_as_names_from_arin(self
, downloader
):
1303 # Delete all previously imported content
1304 self
.db
.execute("DELETE FROM autnums WHERE source = %s", "ARIN")
1306 # Try to retrieve the feed from ftp.arin.net
1307 feed
= downloader
.request_lines("https://ftp.arin.net/pub/resource_registry_service/asns.csv")
1309 # Walk through the file
1310 for line
in csv
.DictReader(feed
, dialect
="arin"):
1311 log
.debug("Processing object: %s" % line
)
1314 status
= line
.get("Status")
1316 # We are only interested in anything managed by ARIN
1317 if not status
== "Full Registry Services":
1320 # Fetch organization name
1321 name
= line
.get("Org Name")
1324 first_asn
= line
.get("Start AS Number")
1325 last_asn
= line
.get("End AS Number")
1329 first_asn
= int(first_asn
)
1330 except TypeError as e
:
1331 log
.warning("Could not parse ASN '%s'" % first_asn
)
1335 last_asn
= int(last_asn
)
1336 except TypeError as e
:
1337 log
.warning("Could not parse ASN '%s'" % last_asn
)
1340 # Check if the range is valid
1341 if last_asn
< first_asn
:
1342 log
.warning("Invalid ASN range %s-%s" % (first_asn
, last_asn
))
1344 # Insert everything into the database
1345 for asn
in range(first_asn
, last_asn
+ 1):
1346 if not self
._check
_parsed
_asn
(asn
):
1347 log
.warning("Skipping invalid ASN %s" % asn
)
1367 """, asn
, name
, "ARIN",
1370 def handle_update_announcements(self
, ns
):
1371 server
= ns
.server
[0]
1373 with self
.db
.transaction():
1374 if server
.startswith("/"):
1375 self
._handle
_update
_announcements
_from
_bird
(server
)
1377 # Purge anything we never want here
1379 -- Delete default routes
1380 DELETE FROM announcements WHERE network = '::/0' OR network = '0.0.0.0/0';
1382 -- Delete anything that is not global unicast address space
1383 DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3';
1385 -- DELETE "current network" address space
1386 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '0.0.0.0/8';
1388 -- DELETE local loopback address space
1389 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '127.0.0.0/8';
1391 -- DELETE RFC 1918 address space
1392 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8';
1393 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12';
1394 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16';
1396 -- DELETE test, benchmark and documentation address space
1397 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.0.0/24';
1398 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.2.0/24';
1399 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.18.0.0/15';
1400 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.51.100.0/24';
1401 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '203.0.113.0/24';
1403 -- DELETE CGNAT address space (RFC 6598)
1404 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '100.64.0.0/10';
1406 -- DELETE link local address space
1407 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '169.254.0.0/16';
1409 -- DELETE IPv6 to IPv4 (6to4) address space (RFC 3068)
1410 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.88.99.0/24';
1411 DELETE FROM announcements WHERE family(network) = 6 AND network <<= '2002::/16';
1413 -- DELETE multicast and reserved address space
1414 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '224.0.0.0/4';
1415 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '240.0.0.0/4';
1417 -- Delete networks that are too small to be in the global routing table
1418 DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48;
1419 DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24;
1421 -- Delete any non-public or reserved ASNs
1422 DELETE FROM announcements WHERE NOT (
1423 (autnum >= 1 AND autnum <= 23455)
1425 (autnum >= 23457 AND autnum <= 64495)
1427 (autnum >= 131072 AND autnum <= 4199999999)
1430 -- Delete everything that we have not seen for 14 days
1431 DELETE FROM announcements WHERE last_seen_at <= CURRENT_TIMESTAMP - INTERVAL '14 days';
1434 def _handle_update_announcements_from_bird(self
, server
):
1435 # Pre-compile the regular expression for faster searching
1436 route
= re
.compile(b
"^\s(.+?)\s+.+?\[(?:AS(.*?))?.\]$")
1438 log
.info("Requesting routing table from Bird (%s)" % server
)
1440 aggregated_networks
= []
1442 # Send command to list all routes
1443 for line
in self
._bird
_cmd
(server
, "show route"):
1444 m
= route
.match(line
)
1450 # Ignore any header lines with the name of the routing table
1451 elif line
.startswith(b
"Table"):
1456 log
.debug("Could not parse line: %s" % line
.decode())
1460 # Fetch the extracted network and ASN
1461 network
, autnum
= m
.groups()
1463 # Decode into strings
1465 network
= network
.decode()
1467 autnum
= autnum
.decode()
1469 # Collect all aggregated networks
1471 log
.debug("%s is an aggregated network" % network
)
1472 aggregated_networks
.append(network
)
1475 # Insert it into the database
1476 self
.db
.execute("INSERT INTO announcements(network, autnum) \
1477 VALUES(%s, %s) ON CONFLICT (network) DO \
1478 UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
1482 # Process any aggregated networks
1483 for network
in aggregated_networks
:
1484 log
.debug("Processing aggregated network %s" % network
)
1486 # Run "show route all" for each network
1487 for line
in self
._bird
_cmd
(server
, "show route %s all" % network
):
1488 # Try finding the path
1489 m
= re
.match(b
"\s+BGP\.as_path:.* (\d+) {\d+}$", line
)
1491 # Select the last AS number in the path
1492 autnum
= m
.group(1).decode()
1494 # Insert it into the database
1495 self
.db
.execute("INSERT INTO announcements(network, autnum) \
1496 VALUES(%s, %s) ON CONFLICT (network) DO \
1497 UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
1501 # We don't need to process any more
1504 def _bird_cmd(self
, socket_path
, command
):
1505 # Connect to the socket
1506 s
= socket
.socket(socket
.AF_UNIX
, socket
.SOCK_STREAM
)
1507 s
.connect(socket_path
)
1509 # Allocate some buffer
1512 log
.debug("Sending Bird command: %s" % command
)
1515 s
.send(b
"%s\n" % command
.encode())
1518 # Fill up the buffer
1519 buffer += s
.recv(4096)
1522 # Search for the next newline
1523 pos
= buffer.find(b
"\n")
1525 # If we cannot find one, we go back and read more data
1529 # Cut after the newline character
1532 # Split the line we want and keep the rest in buffer
1533 line
, buffer = buffer[:pos
], buffer[pos
:]
1535 # Try parsing any status lines
1536 if len(line
) > 4 and line
[:4].isdigit() and line
[4] in (32, 45):
1537 code
, delim
, line
= int(line
[:4]), line
[4], line
[5:]
1539 log
.debug("Received response code %s from bird" % code
)
1549 # Otherwise return the line
1552 def handle_update_geofeeds(self
, ns
):
1554 with self
.db
.transaction():
1555 # Delete all geofeeds which are no longer linked
1566 geofeeds.url = network_geofeeds.url
1585 # Fetch all Geofeeds that require an update
1586 geofeeds
= self
.db
.query("""
1595 updated_at <= CURRENT_TIMESTAMP - INTERVAL '1 week'
1600 # Create a downloader
1601 downloader
= location
.importer
.Downloader()
1603 # Pass the downloader to the fetch_geofeed function
1604 fetch_geofeed
= functools
.partial(self
._fetch
_geofeed
, downloader
)
1606 with concurrent
.futures
.ThreadPoolExecutor(max_workers
=10) as executor
:
1607 results
= executor
.map(fetch_geofeed
, geofeeds
)
1609 # Fetch all results to raise any exceptions
1610 for result
in results
:
1613 # Delete data from any feeds that did not update in the last two weeks
1614 with self
.db
.transaction():
1619 geofeed_networks.geofeed_id IN (
1627 updated_at <= CURRENT_TIMESTAMP - INTERVAL '2 weeks'
1631 def _fetch_geofeed(self
, downloader
, geofeed
):
1632 log
.debug("Fetching Geofeed %s" % geofeed
.url
)
1634 with self
.db
.transaction():
1638 f
= downloader
.retrieve(geofeed
.url
, headers
={
1639 "User-Agent" : "location/%s" % location
.__version
__,
1641 # We expect some plain text file in CSV format
1642 "Accept" : "text/csv, text/plain",
1645 # Remove any previous data
1646 self
.db
.execute("DELETE FROM geofeed_networks \
1647 WHERE geofeed_id = %s", geofeed
.id)
1651 # Read the output line by line
1656 line
= line
.decode()
1658 # Ignore any lines we cannot decode
1659 except UnicodeDecodeError:
1660 log
.debug("Could not decode line %s in %s" \
1661 % (lineno
, geofeed
.url
))
1665 line
= line
.rstrip()
1671 # Try to parse the line
1673 fields
= line
.split(",", 5)
1675 log
.debug("Could not parse line: %s" % line
)
1678 # Check if we have enough fields
1680 log
.debug("Not enough fields in line: %s" % line
)
1684 network
, country
, region
, city
, = fields
[:4]
1686 # Try to parse the network
1688 network
= ipaddress
.ip_network(network
, strict
=False)
1690 log
.debug("Could not parse network: %s" % network
)
1693 # Strip any excess whitespace from country codes
1694 country
= country
.strip()
1696 # Make the country code uppercase
1697 country
= country
.upper()
1699 # Check the country code
1701 log
.debug("Empty country code in Geofeed %s line %s" \
1702 % (geofeed
.url
, lineno
))
1705 elif not location
.country_code_is_valid(country
):
1706 log
.debug("Invalid country code in Geofeed %s:%s: %s" \
1707 % (geofeed
.url
, lineno
, country
))
1710 # Write this into the database
1720 VALUES (%s, %s, %s, %s, %s)""",
1728 # Catch any HTTP errors
1729 except urllib
.request
.HTTPError
as e
:
1730 self
.db
.execute("UPDATE geofeeds SET status = %s, error = %s \
1731 WHERE id = %s", e
.code
, "%s" % e
, geofeed
.id)
1733 # Remove any previous data when the feed has been deleted
1735 self
.db
.execute("DELETE FROM geofeed_networks \
1736 WHERE geofeed_id = %s", geofeed
.id)
1738 # Catch any other errors and connection timeouts
1739 except (http
.client
.InvalidURL
, urllib
.request
.URLError
, TimeoutError
) as e
:
1740 log
.debug("Could not fetch URL %s: %s" % (geofeed
.url
, e
))
1742 self
.db
.execute("UPDATE geofeeds SET status = %s, error = %s \
1743 WHERE id = %s", 599, "%s" % e
, geofeed
.id)
1745 # Mark the geofeed as updated
1751 updated_at = CURRENT_TIMESTAMP,
1759 def handle_update_overrides(self
, ns
):
1760 with self
.db
.transaction():
1761 # Drop any previous content
1762 self
.db
.execute("TRUNCATE TABLE autnum_overrides")
1763 self
.db
.execute("TRUNCATE TABLE network_overrides")
1765 for file in ns
.files
:
1766 log
.info("Reading %s..." % file)
1768 with
open(file, "rb") as f
:
1769 for type, block
in location
.importer
.read_blocks(f
):
1771 network
= block
.get("net")
1772 # Try to parse and normalise the network
1774 network
= ipaddress
.ip_network(network
, strict
=False)
1775 except ValueError as e
:
1776 log
.warning("Invalid IP network: %s: %s" % (network
, e
))
1779 # Prevent that we overwrite all networks
1780 if network
.prefixlen
== 0:
1781 log
.warning("Skipping %s: You cannot overwrite default" % network
)
1791 is_satellite_provider,
1797 %s, %s, %s, %s, %s, %s
1799 ON CONFLICT (network) DO NOTHING
1802 block
.get("country"),
1803 self
._parse
_bool
(block
, "is-anonymous-proxy"),
1804 self
._parse
_bool
(block
, "is-satellite-provider"),
1805 self
._parse
_bool
(block
, "is-anycast"),
1806 self
._parse
_bool
(block
, "drop"),
1809 elif type == "aut-num":
1810 autnum
= block
.get("aut-num")
1812 # Check if AS number begins with "AS"
1813 if not autnum
.startswith("AS"):
1814 log
.warning("Invalid AS number: %s" % autnum
)
1828 is_satellite_provider,
1834 %s, %s, %s, %s, %s, %s, %s
1836 ON CONFLICT (number) DO NOTHING
1840 block
.get("country"),
1841 self
._parse
_bool
(block
, "is-anonymous-proxy"),
1842 self
._parse
_bool
(block
, "is-satellite-provider"),
1843 self
._parse
_bool
(block
, "is-anycast"),
1844 self
._parse
_bool
(block
, "drop"),
1848 log
.warning("Unsupported type: %s" % type)
1850 def handle_update_feeds(self
, ns
):
1852 Update any third-party feeds
1856 # Create a downloader
1857 downloader
= location
.importer
.Downloader()
1861 ("AWS-IP-RANGES", self
._import
_aws
_ip
_ranges
, "https://ip-ranges.amazonaws.com/ip-ranges.json"),
1864 ("SPAMHAUS-DROP", self
._import
_spamhaus
_drop
, "https://www.spamhaus.org/drop/drop.txt"),
1865 ("SPAMHAUS-EDROP", self
._import
_spamhaus
_drop
, "https://www.spamhaus.org/drop/edrop.txt"),
1866 ("SPAMHAUS-DROPV6", self
._import
_spamhaus
_drop
, "https://www.spamhaus.org/drop/dropv6.txt"),
1869 ("SPAMHAUS-ASNDROP", self
._import
_spamhaus
_asndrop
, "https://www.spamhaus.org/drop/asndrop.json"),
1872 # Drop any data from feeds that we don't support (any more)
1873 with self
.db
.transaction():
1874 # Fetch the names of all feeds we support
1875 sources
= [name
for name
, *rest
in feeds
]
1877 self
.db
.execute("DELETE FROM autnum_feeds WHERE NOT source = ANY(%s)", sources
)
1878 self
.db
.execute("DELETE FROM network_feeds WHERE NOT source = ANY(%s)", sources
)
1880 # Walk through all feeds
1881 for name
, callback
, url
, *args
in feeds
:
1882 # Skip any feeds that were not requested on the command line
1883 if ns
.feeds
and not name
in ns
.feeds
:
1887 self
._process
_feed
(downloader
, name
, callback
, url
, *args
)
1889 # Log an error but continue if an exception occurs
1890 except Exception as e
:
1891 log
.error("Error processing feed '%s': %s" % (name
, e
))
1895 return 0 if success
else 1
1897 def _process_feed(self
, downloader
, name
, callback
, url
, *args
):
1902 f
= downloader
.retrieve(url
)
1904 with self
.db
.transaction():
1905 # Drop any previous content
1906 self
.db
.execute("DELETE FROM autnum_feeds WHERE source = %s", name
)
1907 self
.db
.execute("DELETE FROM network_feeds WHERE source = %s", name
)
1909 # Call the callback to process the feed
1910 return callback(name
, f
, *args
)
1912 def _import_aws_ip_ranges(self
, name
, f
):
1916 # Set up a dictionary for mapping a region name to a country. Unfortunately,
1917 # there seems to be no machine-readable version available of this other than
1918 # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html
1919 # (worse, it seems to be incomplete :-/ ); https://www.cloudping.cloud/endpoints
1920 # was helpful here as well.
1921 aws_region_country_map
= {
1923 "af-south-1" : "ZA",
1926 "il-central-1" : "IL", # Tel Aviv
1929 "ap-northeast-1" : "JP",
1930 "ap-northeast-2" : "KR",
1931 "ap-northeast-3" : "JP",
1933 "ap-south-1" : "IN",
1934 "ap-south-2" : "IN",
1935 "ap-southeast-1" : "SG",
1936 "ap-southeast-2" : "AU",
1937 "ap-southeast-3" : "MY",
1938 "ap-southeast-4" : "AU",
1939 "ap-southeast-5" : "NZ", # Auckland, NZ
1940 "ap-southeast-6" : "AP", # XXX: Precise location not documented anywhere
1943 "ca-central-1" : "CA",
1947 "eu-central-1" : "DE",
1948 "eu-central-2" : "CH",
1949 "eu-north-1" : "SE",
1953 "eu-south-1" : "IT",
1954 "eu-south-2" : "ES",
1957 "me-central-1" : "AE",
1958 "me-south-1" : "BH",
1963 # Undocumented, likely located in Berlin rather than Frankfurt
1964 "eusc-de-east-1" : "DE",
1967 # Collect a list of all networks
1968 prefixes
= feed
.get("ipv6_prefixes", []) + feed
.get("prefixes", [])
1970 for prefix
in prefixes
:
1972 network
= prefix
.get("ipv6_prefix") or prefix
.get("ip_prefix")
1976 network
= ipaddress
.ip_network(network
)
1977 except ValuleError
as e
:
1978 log
.warning("%s: Unable to parse prefix %s" % (name
, network
))
1981 # Sanitize parsed networks...
1982 if not self
._check
_parsed
_network
(network
):
1986 region
= prefix
.get("region")
1992 # Fetch the CC from the dictionary
1994 cc
= aws_region_country_map
[region
]
1996 # If we couldn't find anything, let's try something else...
1997 except KeyError as e
:
1998 # Find anycast networks
1999 if region
== "GLOBAL":
2002 # Everything that starts with us- is probably in the United States
2003 elif region
.startswith("us-"):
2006 # Everything that starts with cn- is probably China
2007 elif region
.startswith("cn-"):
2010 # Log a warning for anything else
2012 log
.warning("%s: Could not determine country code for AWS region %s" \
2030 ON CONFLICT (network, source) DO NOTHING
2031 """, "%s" % network
, name
, cc
, is_anycast
,
2034 def _import_spamhaus_drop(self
, name
, f
):
2036 Import Spamhaus DROP IP feeds
2041 # Walk through all lines
2044 line
= line
.decode("utf-8")
2046 # Strip off any comments
2047 line
, _
, comment
= line
.partition(";")
2049 # Ignore empty lines
2053 # Strip any excess whitespace
2056 # Increment line counter
2061 network
= ipaddress
.ip_network(line
)
2062 except ValueError as e
:
2063 log
.warning("%s: Could not parse network: %s - %s" % (name
, line
, e
))
2067 if not self
._check
_parsed
_network
(network
):
2068 log
.warning("%s: Skipping bogus network: %s" % (name
, network
))
2071 # Insert into the database
2083 )""", "%s" % network
, name
, True,
2086 # Raise an exception if we could not import anything
2088 raise RuntimeError("Received bogus feed %s with no data" % name
)
2090 def _import_spamhaus_asndrop(self
, name
, f
):
2092 Import Spamhaus ASNDROP feed
2096 line
= line
.decode("utf-8")
2100 line
= json
.loads(line
)
2101 except json
.JSONDecodeError
as e
:
2102 log
.warning("%s: Unable to parse JSON object %s: %s" % (name
, line
, e
))
2106 type = line
.get("type")
2109 if type == "metadata":
2113 asn
= line
.get("asn")
2115 # Skip any lines without an ASN
2119 # Filter invalid ASNs
2120 if not self
._check
_parsed
_asn
(asn
):
2121 log
.warning("%s: Skipping bogus ASN %s" % (name
, asn
))
2136 )""", "%s" % asn
, name
, True,
2140 def _parse_bool(block
, key
):
2141 val
= block
.get(key
)
2143 # There is no point to proceed when we got None
2147 # Convert to lowercase
2151 if val
in ("yes", "1"):
2155 if val
in ("no", "0"):
2161 def handle_import_countries(self
, ns
):
2162 with self
.db
.transaction():
2163 # Drop all data that we have
2164 self
.db
.execute("TRUNCATE TABLE countries")
2166 for file in ns
.file:
2168 line
= line
.rstrip()
2170 # Ignore any comments
2171 if line
.startswith("#"):
2175 country_code
, continent_code
, name
= line
.split(maxsplit
=2)
2177 log
.warning("Could not parse line: %s" % line
)
2180 self
.db
.execute("INSERT INTO countries(country_code, name, continent_code) \
2181 VALUES(%s, %s, %s) ON CONFLICT DO NOTHING", country_code
, name
, continent_code
)
2184 def split_line(line
):
2185 key
, colon
, val
= line
.partition(":")
2187 # Strip any excess space
2194 # Run the command line interface