From: Stefan Schantl Date: Wed, 10 Jun 2020 16:01:57 +0000 (+0200) Subject: libloc: Import upstream patches. X-Git-Tag: v2.25-core148~67 X-Git-Url: http://git.ipfire.org/?p=ipfire-2.x.git;a=commitdiff_plain;h=4415b1c351e0cae870e5f81e5043f2743bddc063 libloc: Import upstream patches. Signed-off-by: Stefan Schantl --- diff --git a/lfs/libloc b/lfs/libloc index 8e8d539a75..a78df6bd47 100644 --- a/lfs/libloc +++ b/lfs/libloc @@ -76,6 +76,20 @@ $(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects)) # Add upstream patches. cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-location-downloader-do-not-change-content-of-open-database.patch cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-remove-python-path-overrides-for-debian.patch + cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-location-query-require-at-least-one-flag.patch + cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-location-exporter-do-not-mistake-country-as-for-an-as-number.patch + cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-location-exporter-warn-but-do-not-fail-on-invalid-input.patch + cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-move-location-downloader-functionality-into-location-query.patch + cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-merge-location-downloader-manpage-into-location-query.patch + cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-downloader-rename-user-agent-to-location.patch + cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-rename-location-query-to-location.patch + cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-merge-location-exporter-into-location.patch + cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-remove-accidently-commited-hacks-for-debian.patch + cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-add-option-to-iterate-over-all-contries-and-print-them.patch + cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-export-all-countries-by-default.patch + cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-export-flagged-networks-with-their-faked-country-names-too.patch + cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-database-fix-brocken-search-for-networks-with-flags.patch + cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-adjust-format-to-print-ASes.patch cd $(DIR_APP) && ./autogen.sh cd $(DIR_APP) && ./configure \ diff --git a/src/patches/libloc-0.9.1-add-option-to-iterate-over-all-contries-and-print-them.patch b/src/patches/libloc-0.9.1-add-option-to-iterate-over-all-contries-and-print-them.patch new file mode 100644 index 0000000000..e18a6b0963 --- /dev/null +++ b/src/patches/libloc-0.9.1-add-option-to-iterate-over-all-contries-and-print-them.patch @@ -0,0 +1,214 @@ +commit fa9a3663cb2dfb2490da43f6967f1a3a2948fc8a +Author: Michael Tremer +Date: Fri Jun 5 09:41:28 2020 +0000 + + Add option to iterate over all countries and print them to the console + + Signed-off-by: Michael Tremer + +diff --git a/man/location.txt b/man/location.txt +index 672c2b2..0d70e0b 100644 +--- a/man/location.txt ++++ b/man/location.txt +@@ -13,6 +13,7 @@ location - Query the location database + `location list-networks-by-as ASN` + `location list-networks-by-cc COUNTRY_CODE` + `location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]` ++`location list-countries [--show-name] [--show-continent]` + + == DESCRIPTION + `location` retrieves information from the location database. +@@ -86,6 +87,12 @@ or countries. + + + See above for usage of the '--family' and '--output-format' parameters. + ++'list-countries [--show-name] [--show-continent]':: ++ Lists all countries known to the database. ++ + ++ With the optional parameters '--show-name' and '--show-continent', the name and ++ continent code will be printed, too. ++ + '--help':: + Shows a short help text on using this program. + +diff --git a/src/database.c b/src/database.c +index d919278..8e6c5ab 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -107,6 +107,9 @@ struct loc_database_enumerator { + // Index of the AS we are looking at + unsigned int as_index; + ++ // Index of the country we are looking at ++ unsigned int country_index; ++ + // Network state + struct in6_addr network_address; + struct loc_node_stack network_stack[MAX_STACK_DEPTH]; +@@ -1219,3 +1222,30 @@ LOC_EXPORT int loc_database_enumerator_next_network( + + return 0; + } ++ ++LOC_EXPORT int loc_database_enumerator_next_country( ++ struct loc_database_enumerator* enumerator, struct loc_country** country) { ++ *country = NULL; ++ ++ // Do not do anything if not in country mode ++ if (enumerator->mode != LOC_DB_ENUMERATE_COUNTRIES) ++ return 0; ++ ++ struct loc_database* db = enumerator->db; ++ ++ while (enumerator->country_index < db->countries_count) { ++ // Fetch the next country ++ int r = loc_database_fetch_country(db, country, enumerator->country_index++); ++ if (r) ++ return r; ++ ++ // We do not filter here, so it always is a match ++ return 0; ++ } ++ ++ // Reset the index ++ enumerator->country_index = 0; ++ ++ // We have searched through all of them ++ return 0; ++} +diff --git a/src/libloc.sym b/src/libloc.sym +index e9e8549..9a1e6f0 100644 +--- a/src/libloc.sym ++++ b/src/libloc.sym +@@ -68,6 +68,7 @@ global: + # Database Enumerator + loc_database_enumerator_new; + loc_database_enumerator_next_as; ++ loc_database_enumerator_next_country; + loc_database_enumerator_next_network; + loc_database_enumerator_ref; + loc_database_enumerator_set_asn; +diff --git a/src/loc/database.h b/src/loc/database.h +index ab9ef72..43173dd 100644 +--- a/src/loc/database.h ++++ b/src/loc/database.h +@@ -50,8 +50,9 @@ int loc_database_get_country(struct loc_database* db, + struct loc_country** country, const char* code); + + enum loc_database_enumerator_mode { +- LOC_DB_ENUMERATE_NETWORKS = 1, +- LOC_DB_ENUMERATE_ASES = 2, ++ LOC_DB_ENUMERATE_NETWORKS = 1, ++ LOC_DB_ENUMERATE_ASES = 2, ++ LOC_DB_ENUMERATE_COUNTRIES = 3, + }; + + struct loc_database_enumerator; +@@ -69,5 +70,7 @@ int loc_database_enumerator_next_as( + struct loc_database_enumerator* enumerator, struct loc_as** as); + int loc_database_enumerator_next_network( + struct loc_database_enumerator* enumerator, struct loc_network** network); ++int loc_database_enumerator_next_country( ++ struct loc_database_enumerator* enumerator, struct loc_country** country); + + #endif +diff --git a/src/python/database.c b/src/python/database.c +index 581ed5b..1013a58 100644 +--- a/src/python/database.c ++++ b/src/python/database.c +@@ -316,6 +316,10 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, + return obj; + } + ++static PyObject* Database_countries(DatabaseObject* self) { ++ return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES); ++} ++ + static struct PyMethodDef Database_methods[] = { + { + "get_as", +@@ -364,6 +368,13 @@ static struct PyGetSetDef Database_getsetters[] = { + NULL, + NULL, + }, ++ { ++ "countries", ++ (getter)Database_countries, ++ NULL, ++ NULL, ++ NULL, ++ }, + { + "created_at", + (getter)Database_get_created_at, +@@ -462,6 +473,22 @@ static PyObject* DatabaseEnumerator_next(DatabaseEnumeratorObject* self) { + return obj; + } + ++ // Enumerate all countries ++ struct loc_country* country = NULL; ++ ++ r = loc_database_enumerator_next_country(self->enumerator, &country); ++ if (r) { ++ PyErr_SetFromErrno(PyExc_ValueError); ++ return NULL; ++ } ++ ++ if (country) { ++ PyObject* obj = new_country(&CountryType, country); ++ loc_country_unref(country); ++ ++ return obj; ++ } ++ + // Nothing found, that means the end + PyErr_SetNone(PyExc_StopIteration); + return NULL; +diff --git a/src/python/location.in b/src/python/location.in +index 7614cae..5c1effd 100644 +--- a/src/python/location.in ++++ b/src/python/location.in +@@ -147,6 +147,18 @@ class CLI(object): + choices=location.export.formats.keys(), default="list") + list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags) + ++ # List countries ++ list_countries = subparsers.add_parser("list-countries", ++ help=_("Lists all countries"), ++ ) ++ list_countries.add_argument("--show-name", ++ action="store_true", help=_("Show the name of the country"), ++ ) ++ list_countries.add_argument("--show-continent", ++ action="store_true", help=_("Show the continent"), ++ ) ++ list_countries.set_defaults(func=self.handle_list_countries) ++ + # Export + export = subparsers.add_parser("export", + help=_("Exports data in many formats to load it into packet filters"), +@@ -435,6 +447,24 @@ class CLI(object): + + return cls + ++ def handle_list_countries(self, db, ns): ++ for country in db.countries: ++ line = [ ++ country.code, ++ ] ++ ++ if ns.show_continent: ++ line.append(country.continent_code) ++ ++ if ns.show_name: ++ line.append(country.name) ++ ++ # Format the output ++ line = " ".join(line) ++ ++ # Print the output ++ print(line) ++ + def handle_list_networks_by_as(self, db, ns): + writer = self.__get_output_formatter(ns) + diff --git a/src/patches/libloc-0.9.1-adjust-format-to-print-ASes.patch b/src/patches/libloc-0.9.1-adjust-format-to-print-ASes.patch new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/patches/libloc-0.9.1-database-fix-brocken-search-for-networks-with-flags.patch b/src/patches/libloc-0.9.1-database-fix-brocken-search-for-networks-with-flags.patch new file mode 100644 index 0000000000..4d49740bb9 --- /dev/null +++ b/src/patches/libloc-0.9.1-database-fix-brocken-search-for-networks-with-flags.patch @@ -0,0 +1,24 @@ +commit 864dd22e17f7487a90e165274cf3f7898966028d +Author: Michael Tremer +Date: Fri Jun 5 10:01:47 2020 +0000 + + database: Fix broken search for networks with flags + + The search was ended after the first network. No matter if + it matched, or not. + + Signed-off-by: Michael Tremer + +diff --git a/src/database.c b/src/database.c +index 8e6c5ab..fa1dad0 100644 +--- a/src/database.c ++++ b/src/database.c +@@ -1208,6 +1208,8 @@ LOC_EXPORT int loc_database_enumerator_next_network( + !loc_network_match_flag(*network, enumerator->flags)) { + loc_network_unref(*network); + *network = NULL; ++ ++ continue; + } + + return 0; diff --git a/src/patches/libloc-0.9.1-downloader-rename-user-agent-to-location.patch b/src/patches/libloc-0.9.1-downloader-rename-user-agent-to-location.patch new file mode 100644 index 0000000000..1ec77ed68f --- /dev/null +++ b/src/patches/libloc-0.9.1-downloader-rename-user-agent-to-location.patch @@ -0,0 +1,21 @@ +commit dc1df0f469668ef3dc4e1a9a7623a0ebba2b051e +Author: Michael Tremer +Date: Wed Jun 3 17:15:27 2020 +0000 + + downloader: Change user-agent to location + + Signed-off-by: Michael Tremer + +diff --git a/src/python/downloader.py b/src/python/downloader.py +index c9e6e00..eb28007 100644 +--- a/src/python/downloader.py ++++ b/src/python/downloader.py +@@ -71,7 +71,7 @@ class Downloader(object): + + # Update headers + headers.update({ +- "User-Agent" : "location-downloader/@VERSION@", ++ "User-Agent" : "location/@VERSION@", + }) + + # Set headers diff --git a/src/patches/libloc-0.9.1-export-all-countries-by-default.patch b/src/patches/libloc-0.9.1-export-all-countries-by-default.patch new file mode 100644 index 0000000000..426a2b6ec9 --- /dev/null +++ b/src/patches/libloc-0.9.1-export-all-countries-by-default.patch @@ -0,0 +1,33 @@ +commit 10fa313b392a269e15bdaf316218a114d9b23b55 +Author: Michael Tremer +Date: Fri Jun 5 09:47:36 2020 +0000 + + location(8): Export all countries by default + + Signed-off-by: Michael Tremer + +diff --git a/src/python/location.in b/src/python/location.in +index 5c1effd..6ced5f5 100644 +--- a/src/python/location.in ++++ b/src/python/location.in +@@ -169,7 +169,7 @@ class CLI(object): + export.add_argument("--family", + help=_("Specify address family"), choices=("ipv6", "ipv4"), + ) +- export.add_argument("objects", nargs="+", help=_("List country codes or ASNs to export")) ++ export.add_argument("objects", nargs="*", help=_("List country codes or ASNs to export")) + export.set_defaults(func=self.handle_export) + + args = parser.parse_args() +@@ -539,9 +539,9 @@ class CLI(object): + log.warning("Invalid argument: %s" % object) + continue + ++ # Default to exporting all countries + if not countries and not asns: +- log.error("Nothing to export") +- return 2 ++ countries = ["A1", "A2", "A3"] + [country.code for country in db.countries] + + # Select the output format + writer = self.__get_output_formatter(ns) diff --git a/src/patches/libloc-0.9.1-export-flagged-networks-with-their-faked-country-names-too.patch b/src/patches/libloc-0.9.1-export-flagged-networks-with-their-faked-country-names-too.patch new file mode 100644 index 0000000000..857d1f76da --- /dev/null +++ b/src/patches/libloc-0.9.1-export-flagged-networks-with-their-faked-country-names-too.patch @@ -0,0 +1,54 @@ +commit fae36e81a32717ac43c0ce48702f6ff05b7cd029 +Author: Michael Tremer +Date: Fri Jun 5 09:57:41 2020 +0000 + + export flagged networks with their faked country names, too + + This will lead to some networks showing up twice. Once with + their real country and once with their faked one. + + It is likely that the first one will match. + + Signed-off-by: Michael Tremer + +diff --git a/src/python/export.py b/src/python/export.py +index 69fe964..d0bbe77 100644 +--- a/src/python/export.py ++++ b/src/python/export.py +@@ -23,10 +23,18 @@ import logging + import os + import socket + ++import _location ++ + # Initialise logging + log = logging.getLogger("location.export") + log.propagate = 1 + ++flags = { ++ _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1", ++ _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2", ++ _location.NETWORK_FLAG_ANYCAST : "A3", ++} ++ + class OutputWriter(object): + suffix = "networks" + mode = "w" +@@ -173,6 +181,17 @@ class Exporter(object): + except KeyError: + pass + ++ # Handle flags ++ for flag in flags: ++ if network.has_flag(flag): ++ # Fetch the "fake" country code ++ country = flags[flag] ++ ++ try: ++ writers[country].write(network) ++ except KeyError: ++ pass ++ + # Write everything to the filesystem + for writer in writers.values(): + writer.finish() diff --git a/src/patches/libloc-0.9.1-location-exporter-do-not-mistake-country-as-for-an-as-number.patch b/src/patches/libloc-0.9.1-location-exporter-do-not-mistake-country-as-for-an-as-number.patch new file mode 100644 index 0000000000..0d40e26bc8 --- /dev/null +++ b/src/patches/libloc-0.9.1-location-exporter-do-not-mistake-country-as-for-an-as-number.patch @@ -0,0 +1,36 @@ +commit 141b10999b280b2563580c705d5d23dc4c442deb +Author: Michael Tremer +Date: Wed Jun 3 16:31:44 2020 +0000 + + location-exporter: Do not mistake country AS for an AS number + + Signed-off-by: Michael Tremer + +diff --git a/src/python/location-exporter.in b/src/python/location-exporter.in +index 894bb44..5454561 100644 +--- a/src/python/location-exporter.in ++++ b/src/python/location-exporter.in +@@ -22,6 +22,7 @@ import io + import ipaddress + import logging + import os.path ++import re + import socket + import sys + +@@ -258,12 +259,9 @@ class CLI(object): + families = [ socket.AF_INET6, socket.AF_INET ] + + for object in ns.objects: +- if object.startswith("AS"): +- try: +- object = int(object[2:]) +- except ValueError: +- log.error("Invalid argument: %s" % object) +- return 2 ++ m = re.match("^AS(\d+)$", object) ++ if m: ++ object = int(m.group(1)) + + asns.append(object) + diff --git a/src/patches/libloc-0.9.1-location-exporter-warn-but-do-not-fail-on-invalid-input.patch b/src/patches/libloc-0.9.1-location-exporter-warn-but-do-not-fail-on-invalid-input.patch new file mode 100644 index 0000000000..bff68eb483 --- /dev/null +++ b/src/patches/libloc-0.9.1-location-exporter-warn-but-do-not-fail-on-invalid-input.patch @@ -0,0 +1,27 @@ +commit 92af07adfb1e06fe1b055fbcf5ba61159637cd73 +Author: Michael Tremer +Date: Wed Jun 3 16:33:44 2020 +0000 + + location-exporter: Warn, but do not fail on invalid input + + Signed-off-by: Michael Tremer + +diff --git a/src/python/location-exporter.in b/src/python/location-exporter.in +index 5454561..d82f1d3 100644 +--- a/src/python/location-exporter.in ++++ b/src/python/location-exporter.in +@@ -270,8 +270,12 @@ class CLI(object): + countries.append(object) + + else: +- log.error("Invalid argument: %s" % object) +- return 2 ++ log.warning("Invalid argument: %s" % object) ++ continue ++ ++ if not countries and not asns: ++ log.error("Nothing to export") ++ return 2 + + # Open the database + try: diff --git a/src/patches/libloc-0.9.1-location-query-require-at-least-one-flag.patch b/src/patches/libloc-0.9.1-location-query-require-at-least-one-flag.patch new file mode 100644 index 0000000000..62d9d2f681 --- /dev/null +++ b/src/patches/libloc-0.9.1-location-query-require-at-least-one-flag.patch @@ -0,0 +1,37 @@ +commit 228d0e74ec47c9954d3a0e1da2e1c0fc6c1b518f +Author: Michael Tremer +Date: Wed Jun 3 16:15:24 2020 +0000 + + location-query: Require at least one flag + + Signed-off-by: Michael Tremer + +diff --git a/src/python/location-query.in b/src/python/location-query.in +index 5f05b5c..dfdff8c 100644 +--- a/src/python/location-query.in ++++ b/src/python/location-query.in +@@ -246,7 +246,13 @@ class CLI(object): + args.family = 0 + + # Call function +- ret = args.func(db, args) ++ try: ++ ret = args.func(db, args) ++ ++ # Catch invalid inputs ++ except ValueError as e: ++ sys.stderr.write("%s\n" % e) ++ ret = 2 + + # Return with exit code + if ret: +@@ -451,6 +457,9 @@ class CLI(object): + if ns.anycast: + flags |= location.NETWORK_FLAG_ANYCAST + ++ if not flags: ++ raise ValueError(_("You must at least pass one flag")) ++ + with self.__get_output_formatter(ns) as f: + for n in db.search_networks(flags=flags, family=ns.family): + f.network(n) diff --git a/src/patches/libloc-0.9.1-merge-location-downloader-manpage-into-location-query.patch b/src/patches/libloc-0.9.1-merge-location-downloader-manpage-into-location-query.patch new file mode 100644 index 0000000000..3bb04d9c98 --- /dev/null +++ b/src/patches/libloc-0.9.1-merge-location-downloader-manpage-into-location-query.patch @@ -0,0 +1,140 @@ +commit 889b932aa6172c96872be545af37d351f7c1c705 +Author: Michael Tremer +Date: Wed Jun 3 17:10:35 2020 +0000 + + location-downloader: Merge man page into location-query + + Signed-off-by: Michael Tremer + +diff --git a/Makefile.am b/Makefile.am +index c0b1300..91f0436 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -37,6 +37,9 @@ LIBLOC_CURRENT=0 + LIBLOC_REVISION=0 + LIBLOC_AGE=0 + ++pythondir = $(prefix)/lib/python3/dist-packages ++pyexecdir = $(prefix)/lib/python$(PYTHON_VERSION)/lib-dynload ++ + DISTCHECK_CONFIGURE_FLAGS = \ + --with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir) + +@@ -371,7 +374,6 @@ src_test_signature_LDADD = \ + # ------------------------------------------------------------------------------ + + MANPAGES = \ +- man/location-downloader.8 \ + man/location-query.8 + + MANPAGES_TXT = $(patsubst %.8,%.txt,$(MANPAGES)) +diff --git a/man/location-downloader.txt b/man/location-downloader.txt +deleted file mode 100644 +index d733923..0000000 +--- a/man/location-downloader.txt ++++ /dev/null +@@ -1,61 +0,0 @@ +-= location-downloader(8) +- +-== NAME +-location-downloader - Download a location database +- +-== SYNOPSIS +-[verse] +-`location-downloader update` +- +-== DESCRIPTION +-The `location-downloader` command updates the local version of the +-location database. +- +-== OPTIONS +- +---database FILE:: +--d FILE:: +- The path of the database which is being updated. +- + +- If this option is omitted, the system's database will be opened. +- +---quiet:: +- Enable quiet mode +- +---debug:: +- Enable debugging mode +- +-== COMMANDS +- +-'update':: +- This command will try to update the local database. +- + +- It will terminate with a return code of zero if the database has been +- successfully updated. 1 on error, 2 on invalid call and 3 if the +- database was already the latest version. +- +-'verify':: +- Verifies the downloaded database. +- +-'--help':: +- Shows a short help text on using this program. +- +-'--version':: +- Shows the program's version and exists. +- +-== EXIT CODES +-The 'location-downloader' command will normally exit with code zero. +-If there has been a problem and the requested action could not be performed, +-the exit code is unequal to zero. +- +-== HOW IT WORKS +-The downloader checks a DNS record for the latest version of the database. +-It will then try to download a file with that version from a mirror server. +-If the downloaded file is outdated, the next mirror will be tried until we +-have found a file that is recent enough. +- +-== BUGS +-Please report all bugs to the bugtracker at https://bugzilla.ipfire.org/. +- +-== AUTHORS +-Michael Tremer +diff --git a/man/location-query.txt b/man/location-query.txt +index b91e8e1..acb43cd 100644 +--- a/man/location-query.txt ++++ b/man/location-query.txt +@@ -8,6 +8,8 @@ location-query - Query the location database + `location-query lookup ADDRESS [ADDRESS...]` + `location-query get-as ASN [ASN...]` + `location-query search-as STRING` ++`location-query update` ++`location-query verify` + `location-query list-networks-by-as ASN` + `location-query list-networks-by-cc COUNTRY_CODE` + `location-query list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]` +@@ -47,6 +49,16 @@ or countries. + + + The search will be performed case-insensitively. + ++'update':: ++ This command will try to update the local database. ++ + ++ It will terminate with a return code of zero if the database has been ++ successfully updated. 1 on error, 2 on invalid call and 3 if the ++ database was already the latest version. ++ ++'verify':: ++ Verifies the downloaded database. ++ + 'list-networks-by-as [--family=[ipv6|ipv4]] [--output-format FORMAT] ASN':: + Lists all networks which belong to this Autonomous System. + + +@@ -85,6 +97,12 @@ The 'location-query' command will normally exit with code zero. + If there has been a problem and the requested action could not be performed, + the exit code is unequal to zero. + ++== HOW IT WORKS ++The downloader checks a DNS record for the latest version of the database. ++It will then try to download a file with that version from a mirror server. ++If the downloaded file is outdated, the next mirror will be tried until we ++have found a file that is recent enough. ++ + == BUGS + Please report all bugs to the bugtracker at https://bugzilla.ipfire.org/. + diff --git a/src/patches/libloc-0.9.1-merge-location-exporter-into-location.patch b/src/patches/libloc-0.9.1-merge-location-exporter-into-location.patch new file mode 100644 index 0000000000..cda0cb8e69 --- /dev/null +++ b/src/patches/libloc-0.9.1-merge-location-exporter-into-location.patch @@ -0,0 +1,796 @@ +commit 88ef7e9cd4b3a1a5662c7dc071bd7a44e1242cba +Author: Michael Tremer +Date: Wed Jun 3 18:36:28 2020 +0000 + + Merge location-exporter(8) into location(8) + + Signed-off-by: Michael Tremer + +diff --git a/Makefile.am b/Makefile.am +index 59870b1..9f520cc 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -150,6 +150,7 @@ dist_pkgpython_PYTHON = \ + src/python/__init__.py \ + src/python/database.py \ + src/python/downloader.py \ ++ src/python/export.py \ + src/python/i18n.py \ + src/python/importer.py \ + src/python/logger.py +@@ -239,17 +240,14 @@ uninstall-perl: + + bin_SCRIPTS = \ + src/python/location \ +- src/python/location-exporter \ + src/python/location-importer + + EXTRA_DIST += \ + src/python/location.in \ +- src/python/location-exporter.in \ + src/python/location-importer.in + + CLEANFILES += \ + src/python/location \ +- src/python/location-exporter \ + src/python/location-importer + + # ------------------------------------------------------------------------------ +diff --git a/src/python/export.py b/src/python/export.py +new file mode 100644 +index 0000000..69fe964 +--- /dev/null ++++ b/src/python/export.py +@@ -0,0 +1,185 @@ ++#!/usr/bin/python3 ++############################################################################### ++# # ++# libloc - A library to determine the location of someone on the Internet # ++# # ++# Copyright (C) 2020 IPFire Development Team # ++# # ++# This library is free software; you can redistribute it and/or # ++# modify it under the terms of the GNU Lesser General Public # ++# License as published by the Free Software Foundation; either # ++# version 2.1 of the License, or (at your option) any later version. # ++# # ++# This library is distributed in the hope that it will be useful, # ++# but WITHOUT ANY WARRANTY; without even the implied warranty of # ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # ++# Lesser General Public License for more details. # ++# # ++############################################################################### ++ ++import io ++import ipaddress ++import logging ++import os ++import socket ++ ++# Initialise logging ++log = logging.getLogger("location.export") ++log.propagate = 1 ++ ++class OutputWriter(object): ++ suffix = "networks" ++ mode = "w" ++ ++ def __init__(self, f, prefix=None): ++ self.f, self.prefix = f, prefix ++ ++ # Immediately write the header ++ self._write_header() ++ ++ @classmethod ++ def open(cls, filename, **kwargs): ++ """ ++ Convenience function to open a file ++ """ ++ f = open(filename, cls.mode) ++ ++ return cls(f, **kwargs) ++ ++ def __repr__(self): ++ return "<%s f=%s>" % (self.__class__.__name__, self.f) ++ ++ def _write_header(self): ++ """ ++ The header of the file ++ """ ++ pass ++ ++ def _write_footer(self): ++ """ ++ The footer of the file ++ """ ++ pass ++ ++ def write(self, network): ++ self.f.write("%s\n" % network) ++ ++ def finish(self): ++ """ ++ Called when all data has been written ++ """ ++ self._write_footer() ++ ++ # Close the file ++ self.f.close() ++ ++ ++class IpsetOutputWriter(OutputWriter): ++ """ ++ For ipset ++ """ ++ suffix = "ipset" ++ ++ def _write_header(self): ++ self.f.write("create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.prefix) ++ ++ def write(self, network): ++ self.f.write("add %s %s\n" % (self.prefix, network)) ++ ++ ++class NftablesOutputWriter(OutputWriter): ++ """ ++ For nftables ++ """ ++ suffix = "set" ++ ++ def _write_header(self): ++ self.f.write("define %s = {\n" % self.prefix) ++ ++ def _write_footer(self): ++ self.f.write("}\n") ++ ++ def write(self, network): ++ self.f.write(" %s,\n" % network) ++ ++ ++class XTGeoIPOutputWriter(OutputWriter): ++ """ ++ Formats the output in that way, that it can be loaded by ++ the xt_geoip kernel module from xtables-addons. ++ """ ++ suffix = "iv" ++ mode = "wb" ++ ++ def write(self, network): ++ n = ipaddress.ip_network("%s" % network) ++ ++ for address in (n.network_address, n.broadcast_address): ++ bytes = socket.inet_pton( ++ socket.AF_INET6 if address.version == 6 else socket.AF_INET, ++ "%s" % address, ++ ) ++ ++ self.f.write(bytes) ++ ++ ++formats = { ++ "ipset" : IpsetOutputWriter, ++ "list" : OutputWriter, ++ "nftables" : NftablesOutputWriter, ++ "xt_geoip" : XTGeoIPOutputWriter, ++} ++ ++class Exporter(object): ++ def __init__(self, db, writer): ++ self.db, self.writer = db, writer ++ ++ def export(self, directory, families, countries, asns): ++ for family in families: ++ log.debug("Exporting family %s" % family) ++ ++ writers = {} ++ ++ # Create writers for countries ++ for country_code in countries: ++ filename = self._make_filename( ++ directory, prefix=country_code, suffix=self.writer.suffix, family=family, ++ ) ++ ++ writers[country_code] = self.writer.open(filename, prefix="CC_%s" % country_code) ++ ++ # Create writers for ASNs ++ for asn in asns: ++ filename = self._make_filename( ++ directory, "AS%s" % asn, suffix=self.writer.suffix, family=family, ++ ) ++ ++ writers[asn] = self.writer.open(filename, prefix="AS%s" % asn) ++ ++ # Get all networks that match the family ++ networks = self.db.search_networks(family=family) ++ ++ # Walk through all networks ++ for network in networks: ++ # Write matching countries ++ try: ++ writers[network.country_code].write(network) ++ except KeyError: ++ pass ++ ++ # Write matching ASNs ++ try: ++ writers[network.asn].write(network) ++ except KeyError: ++ pass ++ ++ # Write everything to the filesystem ++ for writer in writers.values(): ++ writer.finish() ++ ++ def _make_filename(self, directory, prefix, suffix, family): ++ filename = "%s.%s%s" % ( ++ prefix, suffix, "6" if family == socket.AF_INET6 else "4" ++ ) ++ ++ return os.path.join(directory, filename) +diff --git a/src/python/location-exporter.in b/src/python/location-exporter.in +deleted file mode 100644 +index d82f1d3..0000000 +--- a/src/python/location-exporter.in ++++ /dev/null +@@ -1,300 +0,0 @@ +-#!/usr/bin/python3 +-############################################################################### +-# # +-# libloc - A library to determine the location of someone on the Internet # +-# # +-# Copyright (C) 2019 IPFire Development Team # +-# # +-# This library is free software; you can redistribute it and/or # +-# modify it under the terms of the GNU Lesser General Public # +-# License as published by the Free Software Foundation; either # +-# version 2.1 of the License, or (at your option) any later version. # +-# # +-# This library is distributed in the hope that it will be useful, # +-# but WITHOUT ANY WARRANTY; without even the implied warranty of # +-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # +-# Lesser General Public License for more details. # +-# # +-############################################################################### +- +-import argparse +-import io +-import ipaddress +-import logging +-import os.path +-import re +-import socket +-import sys +- +-# Load our location module +-import location +-from location.i18n import _ +- +-# Initialise logging +-log = logging.getLogger("location.exporter") +-log.propagate = 1 +- +-class OutputWriter(object): +- suffix = "networks" +- +- def __init__(self, family, country_code=None, asn=None): +- self.family, self.country_code, self.asn = family, country_code, asn +- +- self.f = io.BytesIO() +- +- def write_out(self, directory): +- # Make the output filename +- filename = os.path.join( +- directory, self._make_filename(), +- ) +- +- with open(filename, "wb") as f: +- self._write_header(f) +- +- # Copy all data into the file +- f.write(self.f.getbuffer()) +- +- self._write_footer(f) +- +- def _make_filename(self): +- return "%s.%s%s" % ( +- self.country_code or "AS%s" % self.asn, +- self.suffix, +- "6" if self.family == socket.AF_INET6 else "4" +- ) +- +- @property +- def name(self): +- if self.country_code: +- return "CC_%s" % self.country_code +- +- if self.asn: +- return "AS%s" % self.asn +- +- def _write_header(self, f): +- """ +- The header of the file +- """ +- pass +- +- def _write_footer(self, f): +- """ +- The footer of the file +- """ +- pass +- +- def write(self, network): +- s = "%s\n" % network +- +- self.f.write(s.encode("ascii")) +- +- +-class IpsetOutputWriter(OutputWriter): +- """ +- For ipset +- """ +- suffix = "ipset" +- +- def _write_header(self, f): +- h = "create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.name +- +- f.write(h.encode("ascii")) +- +- def write(self, network): +- s = "add %s %s\n" % (self.name, network) +- +- self.f.write(s.encode("ascii")) +- +- +-class NftablesOutputWriter(OutputWriter): +- """ +- For nftables +- """ +- suffix = "set" +- +- def _write_header(self, f): +- h = "define %s = {\n" % self.name +- +- f.write(h.encode("ascii")) +- +- def _write_footer(self, f): +- f.write(b"}") +- +- def write(self, network): +- s = " %s,\n" % network +- +- self.f.write(s.encode("ascii")) +- +- +-class XTGeoIPOutputWriter(OutputWriter): +- """ +- Formats the output in that way, that it can be loaded by +- the xt_geoip kernel module from xtables-addons. +- """ +- suffix = "iv" +- +- def write(self, network): +- n = ipaddress.ip_network("%s" % network) +- +- for address in (n.network_address, n.broadcast_address): +- bytes = socket.inet_pton( +- socket.AF_INET6 if address.version == 6 else socket.AF_INET, +- "%s" % address, +- ) +- +- self.f.write(bytes) +- +- +-class Exporter(object): +- def __init__(self, db, writer): +- self.db = db +- self.writer = writer +- +- def export(self, directory, families, countries, asns): +- for family in families: +- log.debug("Exporting family %s" % family) +- +- writers = {} +- +- # Create writers for countries +- for country_code in countries: +- writers[country_code] = self.writer(family, country_code=country_code) +- +- # Create writers for ASNs +- for asn in asns: +- writers[asn] = self.writer(family, asn=asn) +- +- # Get all networks that match the family +- networks = self.db.search_networks(family=family) +- +- # Walk through all networks +- for network in networks: +- # Write matching countries +- if network.country_code in countries: +- writers[network.country_code].write(network) +- +- # Write matching ASNs +- if network.asn in asns: +- writers[network.asn].write(network) +- +- # Write everything to the filesystem +- for writer in writers.values(): +- writer.write_out(directory) +- +- +-class CLI(object): +- output_formats = { +- "ipset" : IpsetOutputWriter, +- "list" : OutputWriter, +- "nftables" : NftablesOutputWriter, +- "xt_geoip" : XTGeoIPOutputWriter, +- } +- +- def parse_cli(self): +- parser = argparse.ArgumentParser( +- description=_("Location Exporter Command Line Interface"), +- ) +- +- # Global configuration flags +- parser.add_argument("--debug", action="store_true", +- help=_("Enable debug output")) +- parser.add_argument("--quiet", action="store_true", +- help=_("Enable quiet mode")) +- +- # version +- parser.add_argument("--version", action="version", +- version="%(prog)s @VERSION@") +- +- # database +- parser.add_argument("--database", "-d", +- default="@databasedir@/database.db", help=_("Path to database"), +- ) +- +- # format +- parser.add_argument("--format", help=_("Output format"), +- default="list", choices=self.output_formats.keys()) +- +- # directory +- parser.add_argument("--directory", help=_("Output directory"), required=True) +- +- # family +- parser.add_argument("--family", help=_("Specify address family"), choices=("ipv6", "ipv4")) +- +- # Countries and Autonomous Systems +- parser.add_argument("objects", nargs="+") +- +- args = parser.parse_args() +- +- # Configure logging +- if args.debug: +- location.logger.set_level(logging.DEBUG) +- elif args.quiet: +- location.logger.set_level(logging.WARNING) +- +- return args +- +- def run(self): +- # Parse command line arguments +- args = self.parse_cli() +- +- # Call function +- ret = self.handle_export(args) +- +- # Return with exit code +- if ret: +- sys.exit(ret) +- +- # Otherwise just exit +- sys.exit(0) +- +- def handle_export(self, ns): +- countries, asns = [], [] +- +- # Translate family +- if ns.family == "ipv6": +- families = [ socket.AF_INET6 ] +- elif ns.family == "ipv4": +- families = [ socket.AF_INET ] +- else: +- families = [ socket.AF_INET6, socket.AF_INET ] +- +- for object in ns.objects: +- m = re.match("^AS(\d+)$", object) +- if m: +- object = int(m.group(1)) +- +- asns.append(object) +- +- elif location.country_code_is_valid(object) \ +- or object in ("A1", "A2", "A3"): +- countries.append(object) +- +- else: +- log.warning("Invalid argument: %s" % object) +- continue +- +- if not countries and not asns: +- log.error("Nothing to export") +- return 2 +- +- # Open the database +- try: +- db = location.Database(ns.database) +- except FileNotFoundError as e: +- log.error("Count not open database: %s" % ns.database) +- return 1 +- +- # Select the output format +- writer = self.output_formats.get(ns.format) +- assert writer +- +- e = Exporter(db, writer) +- e.export(ns.directory, countries=countries, asns=asns, families=families) +- +- +-def main(): +- # Run the command line interface +- c = CLI() +- c.run() +- +-main() +diff --git a/src/python/location.in b/src/python/location.in +index 10618e2..7614cae 100644 +--- a/src/python/location.in ++++ b/src/python/location.in +@@ -22,6 +22,7 @@ import datetime + import ipaddress + import logging + import os ++import re + import shutil + import socket + import sys +@@ -30,6 +31,8 @@ import time + # Load our location module + import location + import location.downloader ++import location.export ++ + from location.i18n import _ + + # Setup logging +@@ -37,88 +40,7 @@ log = logging.getLogger("location") + + # Output formatters + +-class OutputFormatter(object): +- def __init__(self, ns): +- self.ns = ns +- +- def __enter__(self): +- # Open the output +- self.open() +- +- return self +- +- def __exit__(self, type, value, tb): +- if tb is None: +- self.close() +- +- @property +- def name(self): +- if "country_code" in self.ns: +- return "networks_country_%s" % self.ns.country_code[0] +- +- elif "asn" in self.ns: +- return "networks_AS%s" % self.ns.asn[0] +- +- def open(self): +- pass +- +- def close(self): +- pass +- +- def network(self, network): +- print(network) +- +- +-class IpsetOutputFormatter(OutputFormatter): +- """ +- For nftables +- """ +- def open(self): +- print("create %s hash:net family inet hashsize 1024 maxelem 65536" % self.name) +- +- def network(self, network): +- print("add %s %s" % (self.name, network)) +- +- +-class NftablesOutputFormatter(OutputFormatter): +- """ +- For nftables +- """ +- def open(self): +- print("define %s = {" % self.name) +- +- def close(self): +- print("}") +- +- def network(self, network): +- print(" %s," % network) +- +- +-class XTGeoIPOutputFormatter(OutputFormatter): +- """ +- Formats the output in that way, that it can be loaded by +- the xt_geoip kernel module from xtables-addons. +- """ +- def network(self, network): +- n = ipaddress.ip_network("%s" % network) +- +- for address in (n.network_address, n.broadcast_address): +- bytes = socket.inet_pton( +- socket.AF_INET6 if address.version == 6 else socket.AF_INET, +- "%s" % address, +- ) +- +- os.write(1, bytes) +- +- + class CLI(object): +- output_formats = { +- "ipset" : IpsetOutputFormatter, +- "list" : OutputFormatter, +- "nftables" : NftablesOutputFormatter, +- "xt_geoip" : XTGeoIPOutputFormatter, +- } +- + def parse_cli(self): + parser = argparse.ArgumentParser( + description=_("Location Database Command Line Interface"), +@@ -193,8 +115,8 @@ class CLI(object): + ) + list_networks_by_as.add_argument("asn", nargs=1, type=int) + list_networks_by_as.add_argument("--family", choices=("ipv6", "ipv4")) +- list_networks_by_as.add_argument("--output-format", +- choices=self.output_formats.keys(), default="list") ++ list_networks_by_as.add_argument("--format", ++ choices=location.export.formats.keys(), default="list") + list_networks_by_as.set_defaults(func=self.handle_list_networks_by_as) + + # List all networks in a country +@@ -203,8 +125,8 @@ class CLI(object): + ) + list_networks_by_cc.add_argument("country_code", nargs=1) + list_networks_by_cc.add_argument("--family", choices=("ipv6", "ipv4")) +- list_networks_by_cc.add_argument("--output-format", +- choices=self.output_formats.keys(), default="list") ++ list_networks_by_cc.add_argument("--format", ++ choices=location.export.formats.keys(), default="list") + list_networks_by_cc.set_defaults(func=self.handle_list_networks_by_cc) + + # List all networks with flags +@@ -221,10 +143,23 @@ class CLI(object): + action="store_true", help=_("Anycasts"), + ) + list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4")) +- list_networks_by_flags.add_argument("--output-format", +- choices=self.output_formats.keys(), default="list") ++ list_networks_by_flags.add_argument("--format", ++ choices=location.export.formats.keys(), default="list") + list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags) + ++ # Export ++ export = subparsers.add_parser("export", ++ help=_("Exports data in many formats to load it into packet filters"), ++ ) ++ export.add_argument("--format", help=_("Output format"), ++ choices=location.export.formats.keys(), default="list") ++ export.add_argument("--directory", help=_("Output directory"), required=True) ++ export.add_argument("--family", ++ help=_("Specify address family"), choices=("ipv6", "ipv4"), ++ ) ++ export.add_argument("objects", nargs="+", help=_("List country codes or ASNs to export")) ++ export.set_defaults(func=self.handle_export) ++ + args = parser.parse_args() + + # Configure logging +@@ -494,25 +429,36 @@ class CLI(object): + + def __get_output_formatter(self, ns): + try: +- cls = self.output_formats[ns.output_format] ++ cls = location.export.formats[ns.format] + except KeyError: +- cls = OutputFormatter ++ cls = location.export.OutputFormatter + +- return cls(ns) ++ return cls + + def handle_list_networks_by_as(self, db, ns): +- with self.__get_output_formatter(ns) as f: +- for asn in ns.asn: +- # Print all matching networks +- for n in db.search_networks(asn=asn, family=ns.family): +- f.network(n) ++ writer = self.__get_output_formatter(ns) ++ ++ for asn in ns.asn: ++ f = writer(sys.stdout, prefix="AS%s" % asn) ++ ++ # Print all matching networks ++ for n in db.search_networks(asn=asn, family=ns.family): ++ f.write(n) ++ ++ f.finish() + + def handle_list_networks_by_cc(self, db, ns): +- with self.__get_output_formatter(ns) as f: +- for country_code in ns.country_code: +- # Print all matching networks +- for n in db.search_networks(country_code=country_code, family=ns.family): +- f.network(n) ++ writer = self.__get_output_formatter(ns) ++ ++ for country_code in ns.country_code: ++ # Open standard output ++ f = writer(sys.stdout, prefix=country_code) ++ ++ # Print all matching networks ++ for n in db.search_networks(country_code=country_code, family=ns.family): ++ f.write(n) ++ ++ f.finish() + + def handle_list_networks_by_flags(self, db, ns): + flags = 0 +@@ -529,9 +475,49 @@ class CLI(object): + if not flags: + raise ValueError(_("You must at least pass one flag")) + +- with self.__get_output_formatter(ns) as f: +- for n in db.search_networks(flags=flags, family=ns.family): +- f.network(n) ++ writer = self.__get_output_formatter(ns) ++ f = writer(sys.stdout, prefix="custom") ++ ++ for n in db.search_networks(flags=flags, family=ns.family): ++ f.write(n) ++ ++ f.finish() ++ ++ def handle_export(self, db, ns): ++ countries, asns = [], [] ++ ++ # Translate family ++ if ns.family == "ipv6": ++ families = [ socket.AF_INET6 ] ++ elif ns.family == "ipv4": ++ families = [ socket.AF_INET ] ++ else: ++ families = [ socket.AF_INET6, socket.AF_INET ] ++ ++ for object in ns.objects: ++ m = re.match("^AS(\d+)$", object) ++ if m: ++ object = int(m.group(1)) ++ ++ asns.append(object) ++ ++ elif location.country_code_is_valid(object) \ ++ or object in ("A1", "A2", "A3"): ++ countries.append(object) ++ ++ else: ++ log.warning("Invalid argument: %s" % object) ++ continue ++ ++ if not countries and not asns: ++ log.error("Nothing to export") ++ return 2 ++ ++ # Select the output format ++ writer = self.__get_output_formatter(ns) ++ ++ e = location.export.Exporter(db, writer) ++ e.export(ns.directory, countries=countries, asns=asns, families=families) + + + def main(): diff --git a/src/patches/libloc-0.9.1-move-location-downloader-functionality-into-location-query.patch b/src/patches/libloc-0.9.1-move-location-downloader-functionality-into-location-query.patch new file mode 100644 index 0000000000..c14093fa53 --- /dev/null +++ b/src/patches/libloc-0.9.1-move-location-downloader-functionality-into-location-query.patch @@ -0,0 +1,383 @@ +commit a6f1e3463d4c2085c203ad58072d7a154b663904 +Author: Michael Tremer +Date: Wed Jun 3 17:06:13 2020 +0000 + + Move location-downloader functionality into location-query + + The commands are very long and confusion. Hence we merge this + all into one command. + + Signed-off-by: Michael Tremer + +diff --git a/Makefile.am b/Makefile.am +index 31869e0..c0b1300 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -146,6 +146,7 @@ CLEANFILES += \ + dist_pkgpython_PYTHON = \ + src/python/__init__.py \ + src/python/database.py \ ++ src/python/downloader.py \ + src/python/i18n.py \ + src/python/importer.py \ + src/python/logger.py +@@ -234,19 +235,16 @@ uninstall-perl: + $(DESTDIR)/$(prefix)/man/man3/Location.3pm + + bin_SCRIPTS = \ +- src/python/location-downloader \ + src/python/location-exporter \ + src/python/location-importer \ + src/python/location-query + + EXTRA_DIST += \ +- src/python/location-downloader.in \ + src/python/location-exporter.in \ + src/python/location-importer.in \ + src/python/location-query.in + + CLEANFILES += \ +- src/python/location-downloader \ + src/python/location-exporter \ + src/python/location-importer \ + src/python/location-query +diff --git a/src/python/location-downloader.in b/src/python/downloader.py +similarity index 60% +rename from src/python/location-downloader.in +rename to src/python/downloader.py +index bf0d682..c9e6e00 100644 +--- a/src/python/location-downloader.in ++++ b/src/python/downloader.py +@@ -3,7 +3,7 @@ + # # + # libloc - A library to determine the location of someone on the Internet # + # # +-# Copyright (C) 2019 IPFire Development Team # ++# Copyright (C) 2020 IPFire Development Team # + # # + # This library is free software; you can redistribute it and/or # + # modify it under the terms of the GNU Lesser General Public # +@@ -17,24 +17,18 @@ + # # + ############################################################################### + +-import argparse +-import datetime + import logging + import lzma + import os + import random +-import shutil + import stat +-import sys + import tempfile + import time + import urllib.error + import urllib.parse + import urllib.request + +-# Load our location module +-import location +-from location.i18n import _ ++from _location import Database, DATABASE_VERSION_LATEST + + DATABASE_FILENAME = "location.db.xz" + MIRRORS = ( +@@ -46,9 +40,11 @@ log = logging.getLogger("location.downloader") + log.propagate = 1 + + class Downloader(object): +- def __init__(self, version, mirrors): ++ def __init__(self, version=DATABASE_VERSION_LATEST, mirrors=None): + self.version = version +- self.mirrors = list(mirrors) ++ ++ # Set mirrors or use defaults ++ self.mirrors = list(mirrors or MIRRORS) + + # Randomize mirrors + random.shuffle(self.mirrors) +@@ -117,9 +113,10 @@ class Downloader(object): + + return res + +- def download(self, url, public_key, timestamp=None, tmpdir=None, **kwargs): +- headers = {} ++ def download(self, public_key, timestamp=None, tmpdir=None, **kwargs): ++ url = "%s/%s" % (self.version, DATABASE_FILENAME) + ++ headers = {} + if timestamp: + headers["If-Modified-Since"] = timestamp.strftime( + "%a, %d %b %Y %H:%M:%S GMT", +@@ -191,7 +188,7 @@ class Downloader(object): + """ + log.debug("Opening downloaded database at %s" % f.name) + +- db = location.Database(f.name) ++ db = Database(f.name) + + # Database is not recent + if timestamp and db.created_at < timestamp.timestamp(): +@@ -208,141 +205,3 @@ class Downloader(object): + return False + + return True +- +- +-class CLI(object): +- def __init__(self): +- # Which version are we downloading? +- self.version = location.DATABASE_VERSION_LATEST +- +- self.downloader = Downloader(version=self.version, mirrors=MIRRORS) +- +- def parse_cli(self): +- parser = argparse.ArgumentParser( +- description=_("Location Downloader Command Line Interface"), +- ) +- subparsers = parser.add_subparsers() +- +- # Global configuration flags +- parser.add_argument("--debug", action="store_true", +- help=_("Enable debug output")) +- parser.add_argument("--quiet", action="store_true", +- help=_("Enable quiet mode")) +- +- # version +- parser.add_argument("--version", action="version", +- version="%(prog)s @VERSION@") +- +- # database +- parser.add_argument("--database", "-d", +- default="@databasedir@/database.db", help=_("Path to database"), +- ) +- +- # public key +- parser.add_argument("--public-key", "-k", +- default="@databasedir@/signing-key.pem", help=_("Public Signing Key"), +- ) +- +- # Update +- update = subparsers.add_parser("update", help=_("Update database")) +- update.set_defaults(func=self.handle_update) +- +- # Verify +- verify = subparsers.add_parser("verify", +- help=_("Verify the downloaded database")) +- verify.set_defaults(func=self.handle_verify) +- +- args = parser.parse_args() +- +- # Configure logging +- if args.debug: +- location.logger.set_level(logging.DEBUG) +- elif args.quiet: +- location.logger.set_level(logging.WARNING) +- +- # Print usage if no action was given +- if not "func" in args: +- parser.print_usage() +- sys.exit(2) +- +- return args +- +- def run(self): +- # Parse command line arguments +- args = self.parse_cli() +- +- # Call function +- ret = args.func(args) +- +- # Return with exit code +- if ret: +- sys.exit(ret) +- +- # Otherwise just exit +- sys.exit(0) +- +- def handle_update(self, ns): +- # Fetch the timestamp we need from DNS +- t = location.discover_latest_version(self.version) +- +- # Parse timestamp into datetime format +- timestamp = datetime.datetime.fromtimestamp(t) if t else None +- +- # Open database +- try: +- db = location.Database(ns.database) +- +- # Check if we are already on the latest version +- if timestamp and db.created_at >= timestamp.timestamp(): +- log.info("Already on the latest version") +- return +- +- except FileNotFoundError as e: +- db = None +- +- # Download the database into the correct directory +- tmpdir = os.path.dirname(ns.database) +- +- # Try downloading a new database +- try: +- t = self.downloader.download("%s/%s" % (self.version, DATABASE_FILENAME), +- public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir) +- +- # If no file could be downloaded, log a message +- except FileNotFoundError as e: +- log.error("Could not download a new database") +- return 1 +- +- # If we have not received a new file, there is nothing to do +- if not t: +- return 3 +- +- # Move temporary file to destination +- shutil.move(t.name, ns.database) +- +- return 0 +- +- def handle_verify(self, ns): +- try: +- db = location.Database(ns.database) +- except FileNotFoundError as e: +- log.error("%s: %s" % (ns.database, e)) +- return 127 +- +- # Verify the database +- with open(ns.public_key, "r") as f: +- if not db.verify(f): +- log.error("Could not verify database") +- return 1 +- +- # Success +- log.debug("Database successfully verified") +- return 0 +- +- +-def main(): +- # Run the command line interface +- c = CLI() +- c.run() +- +-main() +diff --git a/src/python/location-query.in b/src/python/location-query.in +index dfdff8c..0291786 100644 +--- a/src/python/location-query.in ++++ b/src/python/location-query.in +@@ -18,16 +18,23 @@ + ############################################################################### + + import argparse ++import datetime + import ipaddress ++import logging + import os ++import shutil + import socket + import sys + import time + + # Load our location module + import location ++import location.downloader + from location.i18n import _ + ++# Setup logging ++log = logging.getLogger("location") ++ + # Output formatters + + class OutputFormatter(object): +@@ -157,6 +164,15 @@ class CLI(object): + dump.add_argument("output", nargs="?", type=argparse.FileType("w")) + dump.set_defaults(func=self.handle_dump) + ++ # Update ++ update = subparsers.add_parser("update", help=_("Update database")) ++ update.set_defaults(func=self.handle_update) ++ ++ # Verify ++ verify = subparsers.add_parser("verify", ++ help=_("Verify the downloaded database")) ++ verify.set_defaults(func=self.handle_verify) ++ + # Get AS + get_as = subparsers.add_parser("get-as", + help=_("Get information about one or multiple Autonomous Systems"), +@@ -423,6 +439,59 @@ class CLI(object): + for a in db.search_as(query): + print(a) + ++ def handle_update(self, db, ns): ++ # Fetch the timestamp we need from DNS ++ t = location.discover_latest_version() ++ ++ # Parse timestamp into datetime format ++ timestamp = datetime.datetime.fromtimestamp(t) if t else None ++ ++ # Check the version of the local database ++ if db and timestamp and db.created_at >= timestamp.timestamp(): ++ log.info("Already on the latest version") ++ return ++ ++ # Download the database into the correct directory ++ tmpdir = os.path.dirname(ns.database) ++ ++ # Create a downloader ++ d = location.downloader.Downloader() ++ ++ # Try downloading a new database ++ try: ++ t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir) ++ ++ # If no file could be downloaded, log a message ++ except FileNotFoundError as e: ++ log.error("Could not download a new database") ++ return 1 ++ ++ # If we have not received a new file, there is nothing to do ++ if not t: ++ return 3 ++ ++ # Move temporary file to destination ++ shutil.move(t.name, ns.database) ++ ++ return 0 ++ ++ def handle_verify(self, ns): ++ try: ++ db = location.Database(ns.database) ++ except FileNotFoundError as e: ++ log.error("%s: %s" % (ns.database, e)) ++ return 127 ++ ++ # Verify the database ++ with open(ns.public_key, "r") as f: ++ if not db.verify(f): ++ log.error("Could not verify database") ++ return 1 ++ ++ # Success ++ log.debug("Database successfully verified") ++ return 0 ++ + def __get_output_formatter(self, ns): + try: + cls = self.output_formats[ns.output_format] +diff --git a/src/python/locationmodule.c b/src/python/locationmodule.c +index a04cab7..5b72be9 100644 +--- a/src/python/locationmodule.c ++++ b/src/python/locationmodule.c +@@ -50,9 +50,9 @@ static PyObject* set_log_level(PyObject* m, PyObject* args) { + } + + static PyObject* discover_latest_version(PyObject* m, PyObject* args) { +- unsigned int version = 0; ++ unsigned int version = LOC_DATABASE_VERSION_LATEST; + +- if (!PyArg_ParseTuple(args, "i", &version)) ++ if (!PyArg_ParseTuple(args, "|i", &version)) + return NULL; + + time_t t = 0; diff --git a/src/patches/libloc-0.9.1-remove-accidently-commited-hacks-for-debian.patch b/src/patches/libloc-0.9.1-remove-accidently-commited-hacks-for-debian.patch new file mode 100644 index 0000000000..f0df20a6e0 --- /dev/null +++ b/src/patches/libloc-0.9.1-remove-accidently-commited-hacks-for-debian.patch @@ -0,0 +1,22 @@ +commit 6bfde1447d237d2a345b99677c5b74e54cbd5739 +Author: Michael Tremer +Date: Thu Jun 4 10:37:50 2020 +0000 + + Makefile: Remove accidentially committed hacks for Debian + + Signed-off-by: Michael Tremer + +diff --git a/Makefile.am b/Makefile.am +index ef57551..c75839c 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -37,9 +37,6 @@ LIBLOC_CURRENT=0 + LIBLOC_REVISION=0 + LIBLOC_AGE=0 + +-pythondir = $(prefix)/lib/python3/dist-packages +-pyexecdir = $(prefix)/lib/python$(PYTHON_VERSION)/lib-dynload +- + DISTCHECK_CONFIGURE_FLAGS = \ + --with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir) + diff --git a/src/patches/libloc-0.9.1-rename-location-query-to-location.patch b/src/patches/libloc-0.9.1-rename-location-query-to-location.patch new file mode 100644 index 0000000000..4a86013afd --- /dev/null +++ b/src/patches/libloc-0.9.1-rename-location-query-to-location.patch @@ -0,0 +1,111 @@ +commit 1d237439676e8b9ee10a6dde2c64f5ba3a057210 +Author: Michael Tremer +Date: Wed Jun 3 17:21:31 2020 +0000 + + Rename location-query(8) to location(8) + + Signed-off-by: Michael Tremer + +diff --git a/Makefile.am b/Makefile.am +index bf204d8..59870b1 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -238,19 +238,19 @@ uninstall-perl: + $(DESTDIR)/$(prefix)/man/man3/Location.3pm + + bin_SCRIPTS = \ ++ src/python/location \ + src/python/location-exporter \ +- src/python/location-importer \ +- src/python/location-query ++ src/python/location-importer + + EXTRA_DIST += \ ++ src/python/location.in \ + src/python/location-exporter.in \ +- src/python/location-importer.in \ +- src/python/location-query.in ++ src/python/location-importer.in + + CLEANFILES += \ ++ src/python/location \ + src/python/location-exporter \ +- src/python/location-importer \ +- src/python/location-query ++ src/python/location-importer + + # ------------------------------------------------------------------------------ + +@@ -374,7 +374,7 @@ src_test_signature_LDADD = \ + # ------------------------------------------------------------------------------ + + MANPAGES = \ +- man/location-query.8 ++ man/location.8 + + MANPAGES_TXT = $(patsubst %.8,%.txt,$(MANPAGES)) + MANPAGES_HTML = $(patsubst %.txt,%.html,$(MANPAGES_TXT)) +diff --git a/man/location-query.txt b/man/location.txt +similarity index 84% +rename from man/location-query.txt +rename to man/location.txt +index acb43cd..672c2b2 100644 +--- a/man/location-query.txt ++++ b/man/location.txt +@@ -1,21 +1,21 @@ +-= location-query(8) ++= location(8) + + == NAME +-location-query - Query the location database ++location - Query the location database + + == SYNOPSIS + [verse] +-`location-query lookup ADDRESS [ADDRESS...]` +-`location-query get-as ASN [ASN...]` +-`location-query search-as STRING` +-`location-query update` +-`location-query verify` +-`location-query list-networks-by-as ASN` +-`location-query list-networks-by-cc COUNTRY_CODE` +-`location-query list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]` ++`location lookup ADDRESS [ADDRESS...]` ++`location get-as ASN [ASN...]` ++`location search-as STRING` ++`location update` ++`location verify` ++`location list-networks-by-as ASN` ++`location list-networks-by-cc COUNTRY_CODE` ++`location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]` + + == DESCRIPTION +-The `location-query` retrieves information from the location database. ++`location` retrieves information from the location database. + This data can be used to determine someone's location on the Internet + and for building firewall rulesets to block access from certain ASes + or countries. +@@ -93,7 +93,7 @@ or countries. + Shows the program's version and exists. + + == EXIT CODES +-The 'location-query' command will normally exit with code zero. ++The 'location' command will normally exit with code zero. + If there has been a problem and the requested action could not be performed, + the exit code is unequal to zero. + +diff --git a/src/python/location-query.in b/src/python/location.in +similarity index 99% +rename from src/python/location-query.in +rename to src/python/location.in +index 0291786..10618e2 100644 +--- a/src/python/location-query.in ++++ b/src/python/location.in +@@ -248,7 +248,7 @@ class CLI(object): + try: + db = location.Database(args.database) + except FileNotFoundError as e: +- sys.stderr.write("location-query: Could not open database %s: %s\n" \ ++ sys.stderr.write("location: Could not open database %s: %s\n" \ + % (args.database, e)) + sys.exit(1)