From: Stefan Schantl Date: Mon, 15 Jun 2020 17:47:07 +0000 (+0200) Subject: libloc: Update to 0.9.2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=45f4de2bbcafb689b21a201c0aa836881ce417c2;p=people%2Fms%2Fipfire-2.x.git libloc: Update to 0.9.2 Signed-off-by: Stefan Schantl --- diff --git a/lfs/libloc b/lfs/libloc index c4d90ac526..c4e409fa52 100644 --- a/lfs/libloc +++ b/lfs/libloc @@ -24,7 +24,7 @@ include Config -VER = 0.9.1 +VER = 0.9.2 DB_DATE = 2020-06-10 THISAPP = libloc-$(VER) @@ -40,10 +40,10 @@ TARGET = $(DIR_INFO)/$(THISAPP) objects = $(DL_FILE) \ location-$(DB_DATE).db.xz -$(DL_FILE) = $(DL_FROM)/$(DL_FILE) +$(DL_FILE) = https://source.ipfire.org/releases/libloc//$(DL_FILE) location-$(DB_DATE).db.xz = https://location.ipfire.org/databases/1/archive/location-$(DB_DATE).db.xz -$(DL_FILE)_MD5 = b62331e7a5bc5299bdd35f340342fc51 +$(DL_FILE)_MD5 = c6ed4fcbdb2ce4ca7e6df8abbd985411 location-$(DB_DATE).db.xz_MD5 = 268b6d58a26c6d36081ed1e899b89020 install : $(TARGET) @@ -78,24 +78,6 @@ $(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects)) @$(PREBUILD) @rm -rf $(DIR_APP) && cd $(DIR_SRC) && tar xvf $(DIR_DL)/$(DL_FILE) - # Add upstream patches. - cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-location-downloader-do-not-change-content-of-open-database.patch - cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-remove-python-path-overrides-for-debian.patch - cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-location-query-require-at-least-one-flag.patch - cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-location-exporter-do-not-mistake-country-as-for-an-as-number.patch - cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-location-exporter-warn-but-do-not-fail-on-invalid-input.patch - cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-move-location-downloader-functionality-into-location-query.patch - cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-merge-location-downloader-manpage-into-location-query.patch - cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-downloader-rename-user-agent-to-location.patch - cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-rename-location-query-to-location.patch - cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-merge-location-exporter-into-location.patch - cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-remove-accidently-commited-hacks-for-debian.patch - cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-add-option-to-iterate-over-all-contries-and-print-them.patch - cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-export-all-countries-by-default.patch - cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-export-flagged-networks-with-their-faked-country-names-too.patch - cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-database-fix-brocken-search-for-networks-with-flags.patch - cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-adjust-format-to-print-ASes.patch - cd $(DIR_APP) && ./autogen.sh cd $(DIR_APP) && ./configure \ --prefix=/usr \ diff --git a/src/patches/libloc-0.9.1-add-option-to-iterate-over-all-contries-and-print-them.patch b/src/patches/libloc-0.9.1-add-option-to-iterate-over-all-contries-and-print-them.patch deleted file mode 100644 index e18a6b0963..0000000000 --- a/src/patches/libloc-0.9.1-add-option-to-iterate-over-all-contries-and-print-them.patch +++ /dev/null @@ -1,214 +0,0 @@ -commit fa9a3663cb2dfb2490da43f6967f1a3a2948fc8a -Author: Michael Tremer -Date: Fri Jun 5 09:41:28 2020 +0000 - - Add option to iterate over all countries and print them to the console - - Signed-off-by: Michael Tremer - -diff --git a/man/location.txt b/man/location.txt -index 672c2b2..0d70e0b 100644 ---- a/man/location.txt -+++ b/man/location.txt -@@ -13,6 +13,7 @@ location - Query the location database - `location list-networks-by-as ASN` - `location list-networks-by-cc COUNTRY_CODE` - `location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]` -+`location list-countries [--show-name] [--show-continent]` - - == DESCRIPTION - `location` retrieves information from the location database. -@@ -86,6 +87,12 @@ or countries. - + - See above for usage of the '--family' and '--output-format' parameters. - -+'list-countries [--show-name] [--show-continent]':: -+ Lists all countries known to the database. -+ + -+ With the optional parameters '--show-name' and '--show-continent', the name and -+ continent code will be printed, too. -+ - '--help':: - Shows a short help text on using this program. - -diff --git a/src/database.c b/src/database.c -index d919278..8e6c5ab 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -107,6 +107,9 @@ struct loc_database_enumerator { - // Index of the AS we are looking at - unsigned int as_index; - -+ // Index of the country we are looking at -+ unsigned int country_index; -+ - // Network state - struct in6_addr network_address; - struct loc_node_stack network_stack[MAX_STACK_DEPTH]; -@@ -1219,3 +1222,30 @@ LOC_EXPORT int loc_database_enumerator_next_network( - - return 0; - } -+ -+LOC_EXPORT int loc_database_enumerator_next_country( -+ struct loc_database_enumerator* enumerator, struct loc_country** country) { -+ *country = NULL; -+ -+ // Do not do anything if not in country mode -+ if (enumerator->mode != LOC_DB_ENUMERATE_COUNTRIES) -+ return 0; -+ -+ struct loc_database* db = enumerator->db; -+ -+ while (enumerator->country_index < db->countries_count) { -+ // Fetch the next country -+ int r = loc_database_fetch_country(db, country, enumerator->country_index++); -+ if (r) -+ return r; -+ -+ // We do not filter here, so it always is a match -+ return 0; -+ } -+ -+ // Reset the index -+ enumerator->country_index = 0; -+ -+ // We have searched through all of them -+ return 0; -+} -diff --git a/src/libloc.sym b/src/libloc.sym -index e9e8549..9a1e6f0 100644 ---- a/src/libloc.sym -+++ b/src/libloc.sym -@@ -68,6 +68,7 @@ global: - # Database Enumerator - loc_database_enumerator_new; - loc_database_enumerator_next_as; -+ loc_database_enumerator_next_country; - loc_database_enumerator_next_network; - loc_database_enumerator_ref; - loc_database_enumerator_set_asn; -diff --git a/src/loc/database.h b/src/loc/database.h -index ab9ef72..43173dd 100644 ---- a/src/loc/database.h -+++ b/src/loc/database.h -@@ -50,8 +50,9 @@ int loc_database_get_country(struct loc_database* db, - struct loc_country** country, const char* code); - - enum loc_database_enumerator_mode { -- LOC_DB_ENUMERATE_NETWORKS = 1, -- LOC_DB_ENUMERATE_ASES = 2, -+ LOC_DB_ENUMERATE_NETWORKS = 1, -+ LOC_DB_ENUMERATE_ASES = 2, -+ LOC_DB_ENUMERATE_COUNTRIES = 3, - }; - - struct loc_database_enumerator; -@@ -69,5 +70,7 @@ int loc_database_enumerator_next_as( - struct loc_database_enumerator* enumerator, struct loc_as** as); - int loc_database_enumerator_next_network( - struct loc_database_enumerator* enumerator, struct loc_network** network); -+int loc_database_enumerator_next_country( -+ struct loc_database_enumerator* enumerator, struct loc_country** country); - - #endif -diff --git a/src/python/database.c b/src/python/database.c -index 581ed5b..1013a58 100644 ---- a/src/python/database.c -+++ b/src/python/database.c -@@ -316,6 +316,10 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, - return obj; - } - -+static PyObject* Database_countries(DatabaseObject* self) { -+ return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES); -+} -+ - static struct PyMethodDef Database_methods[] = { - { - "get_as", -@@ -364,6 +368,13 @@ static struct PyGetSetDef Database_getsetters[] = { - NULL, - NULL, - }, -+ { -+ "countries", -+ (getter)Database_countries, -+ NULL, -+ NULL, -+ NULL, -+ }, - { - "created_at", - (getter)Database_get_created_at, -@@ -462,6 +473,22 @@ static PyObject* DatabaseEnumerator_next(DatabaseEnumeratorObject* self) { - return obj; - } - -+ // Enumerate all countries -+ struct loc_country* country = NULL; -+ -+ r = loc_database_enumerator_next_country(self->enumerator, &country); -+ if (r) { -+ PyErr_SetFromErrno(PyExc_ValueError); -+ return NULL; -+ } -+ -+ if (country) { -+ PyObject* obj = new_country(&CountryType, country); -+ loc_country_unref(country); -+ -+ return obj; -+ } -+ - // Nothing found, that means the end - PyErr_SetNone(PyExc_StopIteration); - return NULL; -diff --git a/src/python/location.in b/src/python/location.in -index 7614cae..5c1effd 100644 ---- a/src/python/location.in -+++ b/src/python/location.in -@@ -147,6 +147,18 @@ class CLI(object): - choices=location.export.formats.keys(), default="list") - list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags) - -+ # List countries -+ list_countries = subparsers.add_parser("list-countries", -+ help=_("Lists all countries"), -+ ) -+ list_countries.add_argument("--show-name", -+ action="store_true", help=_("Show the name of the country"), -+ ) -+ list_countries.add_argument("--show-continent", -+ action="store_true", help=_("Show the continent"), -+ ) -+ list_countries.set_defaults(func=self.handle_list_countries) -+ - # Export - export = subparsers.add_parser("export", - help=_("Exports data in many formats to load it into packet filters"), -@@ -435,6 +447,24 @@ class CLI(object): - - return cls - -+ def handle_list_countries(self, db, ns): -+ for country in db.countries: -+ line = [ -+ country.code, -+ ] -+ -+ if ns.show_continent: -+ line.append(country.continent_code) -+ -+ if ns.show_name: -+ line.append(country.name) -+ -+ # Format the output -+ line = " ".join(line) -+ -+ # Print the output -+ print(line) -+ - def handle_list_networks_by_as(self, db, ns): - writer = self.__get_output_formatter(ns) - diff --git a/src/patches/libloc-0.9.1-adjust-format-to-print-ASes.patch b/src/patches/libloc-0.9.1-adjust-format-to-print-ASes.patch deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/patches/libloc-0.9.1-database-fix-brocken-search-for-networks-with-flags.patch b/src/patches/libloc-0.9.1-database-fix-brocken-search-for-networks-with-flags.patch deleted file mode 100644 index 4d49740bb9..0000000000 --- a/src/patches/libloc-0.9.1-database-fix-brocken-search-for-networks-with-flags.patch +++ /dev/null @@ -1,24 +0,0 @@ -commit 864dd22e17f7487a90e165274cf3f7898966028d -Author: Michael Tremer -Date: Fri Jun 5 10:01:47 2020 +0000 - - database: Fix broken search for networks with flags - - The search was ended after the first network. No matter if - it matched, or not. - - Signed-off-by: Michael Tremer - -diff --git a/src/database.c b/src/database.c -index 8e6c5ab..fa1dad0 100644 ---- a/src/database.c -+++ b/src/database.c -@@ -1208,6 +1208,8 @@ LOC_EXPORT int loc_database_enumerator_next_network( - !loc_network_match_flag(*network, enumerator->flags)) { - loc_network_unref(*network); - *network = NULL; -+ -+ continue; - } - - return 0; diff --git a/src/patches/libloc-0.9.1-downloader-rename-user-agent-to-location.patch b/src/patches/libloc-0.9.1-downloader-rename-user-agent-to-location.patch deleted file mode 100644 index 1ec77ed68f..0000000000 --- a/src/patches/libloc-0.9.1-downloader-rename-user-agent-to-location.patch +++ /dev/null @@ -1,21 +0,0 @@ -commit dc1df0f469668ef3dc4e1a9a7623a0ebba2b051e -Author: Michael Tremer -Date: Wed Jun 3 17:15:27 2020 +0000 - - downloader: Change user-agent to location - - Signed-off-by: Michael Tremer - -diff --git a/src/python/downloader.py b/src/python/downloader.py -index c9e6e00..eb28007 100644 ---- a/src/python/downloader.py -+++ b/src/python/downloader.py -@@ -71,7 +71,7 @@ class Downloader(object): - - # Update headers - headers.update({ -- "User-Agent" : "location-downloader/@VERSION@", -+ "User-Agent" : "location/@VERSION@", - }) - - # Set headers diff --git a/src/patches/libloc-0.9.1-export-all-countries-by-default.patch b/src/patches/libloc-0.9.1-export-all-countries-by-default.patch deleted file mode 100644 index 426a2b6ec9..0000000000 --- a/src/patches/libloc-0.9.1-export-all-countries-by-default.patch +++ /dev/null @@ -1,33 +0,0 @@ -commit 10fa313b392a269e15bdaf316218a114d9b23b55 -Author: Michael Tremer -Date: Fri Jun 5 09:47:36 2020 +0000 - - location(8): Export all countries by default - - Signed-off-by: Michael Tremer - -diff --git a/src/python/location.in b/src/python/location.in -index 5c1effd..6ced5f5 100644 ---- a/src/python/location.in -+++ b/src/python/location.in -@@ -169,7 +169,7 @@ class CLI(object): - export.add_argument("--family", - help=_("Specify address family"), choices=("ipv6", "ipv4"), - ) -- export.add_argument("objects", nargs="+", help=_("List country codes or ASNs to export")) -+ export.add_argument("objects", nargs="*", help=_("List country codes or ASNs to export")) - export.set_defaults(func=self.handle_export) - - args = parser.parse_args() -@@ -539,9 +539,9 @@ class CLI(object): - log.warning("Invalid argument: %s" % object) - continue - -+ # Default to exporting all countries - if not countries and not asns: -- log.error("Nothing to export") -- return 2 -+ countries = ["A1", "A2", "A3"] + [country.code for country in db.countries] - - # Select the output format - writer = self.__get_output_formatter(ns) diff --git a/src/patches/libloc-0.9.1-export-flagged-networks-with-their-faked-country-names-too.patch b/src/patches/libloc-0.9.1-export-flagged-networks-with-their-faked-country-names-too.patch deleted file mode 100644 index 857d1f76da..0000000000 --- a/src/patches/libloc-0.9.1-export-flagged-networks-with-their-faked-country-names-too.patch +++ /dev/null @@ -1,54 +0,0 @@ -commit fae36e81a32717ac43c0ce48702f6ff05b7cd029 -Author: Michael Tremer -Date: Fri Jun 5 09:57:41 2020 +0000 - - export flagged networks with their faked country names, too - - This will lead to some networks showing up twice. Once with - their real country and once with their faked one. - - It is likely that the first one will match. - - Signed-off-by: Michael Tremer - -diff --git a/src/python/export.py b/src/python/export.py -index 69fe964..d0bbe77 100644 ---- a/src/python/export.py -+++ b/src/python/export.py -@@ -23,10 +23,18 @@ import logging - import os - import socket - -+import _location -+ - # Initialise logging - log = logging.getLogger("location.export") - log.propagate = 1 - -+flags = { -+ _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1", -+ _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2", -+ _location.NETWORK_FLAG_ANYCAST : "A3", -+} -+ - class OutputWriter(object): - suffix = "networks" - mode = "w" -@@ -173,6 +181,17 @@ class Exporter(object): - except KeyError: - pass - -+ # Handle flags -+ for flag in flags: -+ if network.has_flag(flag): -+ # Fetch the "fake" country code -+ country = flags[flag] -+ -+ try: -+ writers[country].write(network) -+ except KeyError: -+ pass -+ - # Write everything to the filesystem - for writer in writers.values(): - writer.finish() diff --git a/src/patches/libloc-0.9.1-location-downloader-do-not-change-content-of-open-database.patch b/src/patches/libloc-0.9.1-location-downloader-do-not-change-content-of-open-database.patch deleted file mode 100644 index 857ab5c586..0000000000 --- a/src/patches/libloc-0.9.1-location-downloader-do-not-change-content-of-open-database.patch +++ /dev/null @@ -1,81 +0,0 @@ -commit 679e5ae2e45b98e254c651cb3102a4331c2c22eb -Author: Michael Tremer -Date: Mon Jun 1 13:47:44 2020 +0000 - - location-downloader: Do not change content of open database files - - The database might be opened by another process. When modified, - it will return random results. - - Fixes: #12420 - Signed-off-by: Michael Tremer - -diff --git a/src/python/location-downloader.in b/src/python/location-downloader.in -index 7d06030..bf0d682 100644 ---- a/src/python/location-downloader.in -+++ b/src/python/location-downloader.in -@@ -24,6 +24,7 @@ import lzma - import os - import random - import shutil -+import stat - import sys - import tempfile - import time -@@ -116,7 +117,7 @@ class Downloader(object): - - return res - -- def download(self, url, public_key, timestamp=None, **kwargs): -+ def download(self, url, public_key, timestamp=None, tmpdir=None, **kwargs): - headers = {} - - if timestamp: -@@ -124,7 +125,7 @@ class Downloader(object): - "%a, %d %b %Y %H:%M:%S GMT", - ) - -- t = tempfile.NamedTemporaryFile(delete=False) -+ t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False) - with t: - # Try all mirrors - for mirror in self.mirrors: -@@ -175,6 +176,9 @@ class Downloader(object): - t.truncate() - continue - -+ # Make the file readable for everyone -+ os.chmod(t.name, stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH) -+ - # Return temporary file - return t - -@@ -296,10 +300,13 @@ class CLI(object): - except FileNotFoundError as e: - db = None - -+ # Download the database into the correct directory -+ tmpdir = os.path.dirname(ns.database) -+ - # Try downloading a new database - try: - t = self.downloader.download("%s/%s" % (self.version, DATABASE_FILENAME), -- public_key=ns.public_key, timestamp=timestamp) -+ public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir) - - # If no file could be downloaded, log a message - except FileNotFoundError as e: -@@ -310,11 +317,8 @@ class CLI(object): - if not t: - return 3 - -- # Write temporary file to destination -- shutil.copyfile(t.name, ns.database) -- -- # Remove temporary file -- os.unlink(t.name) -+ # Move temporary file to destination -+ shutil.move(t.name, ns.database) - - return 0 - diff --git a/src/patches/libloc-0.9.1-location-exporter-do-not-mistake-country-as-for-an-as-number.patch b/src/patches/libloc-0.9.1-location-exporter-do-not-mistake-country-as-for-an-as-number.patch deleted file mode 100644 index 0d40e26bc8..0000000000 --- a/src/patches/libloc-0.9.1-location-exporter-do-not-mistake-country-as-for-an-as-number.patch +++ /dev/null @@ -1,36 +0,0 @@ -commit 141b10999b280b2563580c705d5d23dc4c442deb -Author: Michael Tremer -Date: Wed Jun 3 16:31:44 2020 +0000 - - location-exporter: Do not mistake country AS for an AS number - - Signed-off-by: Michael Tremer - -diff --git a/src/python/location-exporter.in b/src/python/location-exporter.in -index 894bb44..5454561 100644 ---- a/src/python/location-exporter.in -+++ b/src/python/location-exporter.in -@@ -22,6 +22,7 @@ import io - import ipaddress - import logging - import os.path -+import re - import socket - import sys - -@@ -258,12 +259,9 @@ class CLI(object): - families = [ socket.AF_INET6, socket.AF_INET ] - - for object in ns.objects: -- if object.startswith("AS"): -- try: -- object = int(object[2:]) -- except ValueError: -- log.error("Invalid argument: %s" % object) -- return 2 -+ m = re.match("^AS(\d+)$", object) -+ if m: -+ object = int(m.group(1)) - - asns.append(object) - diff --git a/src/patches/libloc-0.9.1-location-exporter-warn-but-do-not-fail-on-invalid-input.patch b/src/patches/libloc-0.9.1-location-exporter-warn-but-do-not-fail-on-invalid-input.patch deleted file mode 100644 index bff68eb483..0000000000 --- a/src/patches/libloc-0.9.1-location-exporter-warn-but-do-not-fail-on-invalid-input.patch +++ /dev/null @@ -1,27 +0,0 @@ -commit 92af07adfb1e06fe1b055fbcf5ba61159637cd73 -Author: Michael Tremer -Date: Wed Jun 3 16:33:44 2020 +0000 - - location-exporter: Warn, but do not fail on invalid input - - Signed-off-by: Michael Tremer - -diff --git a/src/python/location-exporter.in b/src/python/location-exporter.in -index 5454561..d82f1d3 100644 ---- a/src/python/location-exporter.in -+++ b/src/python/location-exporter.in -@@ -270,8 +270,12 @@ class CLI(object): - countries.append(object) - - else: -- log.error("Invalid argument: %s" % object) -- return 2 -+ log.warning("Invalid argument: %s" % object) -+ continue -+ -+ if not countries and not asns: -+ log.error("Nothing to export") -+ return 2 - - # Open the database - try: diff --git a/src/patches/libloc-0.9.1-location-query-require-at-least-one-flag.patch b/src/patches/libloc-0.9.1-location-query-require-at-least-one-flag.patch deleted file mode 100644 index 62d9d2f681..0000000000 --- a/src/patches/libloc-0.9.1-location-query-require-at-least-one-flag.patch +++ /dev/null @@ -1,37 +0,0 @@ -commit 228d0e74ec47c9954d3a0e1da2e1c0fc6c1b518f -Author: Michael Tremer -Date: Wed Jun 3 16:15:24 2020 +0000 - - location-query: Require at least one flag - - Signed-off-by: Michael Tremer - -diff --git a/src/python/location-query.in b/src/python/location-query.in -index 5f05b5c..dfdff8c 100644 ---- a/src/python/location-query.in -+++ b/src/python/location-query.in -@@ -246,7 +246,13 @@ class CLI(object): - args.family = 0 - - # Call function -- ret = args.func(db, args) -+ try: -+ ret = args.func(db, args) -+ -+ # Catch invalid inputs -+ except ValueError as e: -+ sys.stderr.write("%s\n" % e) -+ ret = 2 - - # Return with exit code - if ret: -@@ -451,6 +457,9 @@ class CLI(object): - if ns.anycast: - flags |= location.NETWORK_FLAG_ANYCAST - -+ if not flags: -+ raise ValueError(_("You must at least pass one flag")) -+ - with self.__get_output_formatter(ns) as f: - for n in db.search_networks(flags=flags, family=ns.family): - f.network(n) diff --git a/src/patches/libloc-0.9.1-merge-location-downloader-manpage-into-location-query.patch b/src/patches/libloc-0.9.1-merge-location-downloader-manpage-into-location-query.patch deleted file mode 100644 index 3bb04d9c98..0000000000 --- a/src/patches/libloc-0.9.1-merge-location-downloader-manpage-into-location-query.patch +++ /dev/null @@ -1,140 +0,0 @@ -commit 889b932aa6172c96872be545af37d351f7c1c705 -Author: Michael Tremer -Date: Wed Jun 3 17:10:35 2020 +0000 - - location-downloader: Merge man page into location-query - - Signed-off-by: Michael Tremer - -diff --git a/Makefile.am b/Makefile.am -index c0b1300..91f0436 100644 ---- a/Makefile.am -+++ b/Makefile.am -@@ -37,6 +37,9 @@ LIBLOC_CURRENT=0 - LIBLOC_REVISION=0 - LIBLOC_AGE=0 - -+pythondir = $(prefix)/lib/python3/dist-packages -+pyexecdir = $(prefix)/lib/python$(PYTHON_VERSION)/lib-dynload -+ - DISTCHECK_CONFIGURE_FLAGS = \ - --with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir) - -@@ -371,7 +374,6 @@ src_test_signature_LDADD = \ - # ------------------------------------------------------------------------------ - - MANPAGES = \ -- man/location-downloader.8 \ - man/location-query.8 - - MANPAGES_TXT = $(patsubst %.8,%.txt,$(MANPAGES)) -diff --git a/man/location-downloader.txt b/man/location-downloader.txt -deleted file mode 100644 -index d733923..0000000 ---- a/man/location-downloader.txt -+++ /dev/null -@@ -1,61 +0,0 @@ --= location-downloader(8) -- --== NAME --location-downloader - Download a location database -- --== SYNOPSIS --[verse] --`location-downloader update` -- --== DESCRIPTION --The `location-downloader` command updates the local version of the --location database. -- --== OPTIONS -- ----database FILE:: ---d FILE:: -- The path of the database which is being updated. -- + -- If this option is omitted, the system's database will be opened. -- ----quiet:: -- Enable quiet mode -- ----debug:: -- Enable debugging mode -- --== COMMANDS -- --'update':: -- This command will try to update the local database. -- + -- It will terminate with a return code of zero if the database has been -- successfully updated. 1 on error, 2 on invalid call and 3 if the -- database was already the latest version. -- --'verify':: -- Verifies the downloaded database. -- --'--help':: -- Shows a short help text on using this program. -- --'--version':: -- Shows the program's version and exists. -- --== EXIT CODES --The 'location-downloader' command will normally exit with code zero. --If there has been a problem and the requested action could not be performed, --the exit code is unequal to zero. -- --== HOW IT WORKS --The downloader checks a DNS record for the latest version of the database. --It will then try to download a file with that version from a mirror server. --If the downloaded file is outdated, the next mirror will be tried until we --have found a file that is recent enough. -- --== BUGS --Please report all bugs to the bugtracker at https://bugzilla.ipfire.org/. -- --== AUTHORS --Michael Tremer -diff --git a/man/location-query.txt b/man/location-query.txt -index b91e8e1..acb43cd 100644 ---- a/man/location-query.txt -+++ b/man/location-query.txt -@@ -8,6 +8,8 @@ location-query - Query the location database - `location-query lookup ADDRESS [ADDRESS...]` - `location-query get-as ASN [ASN...]` - `location-query search-as STRING` -+`location-query update` -+`location-query verify` - `location-query list-networks-by-as ASN` - `location-query list-networks-by-cc COUNTRY_CODE` - `location-query list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]` -@@ -47,6 +49,16 @@ or countries. - + - The search will be performed case-insensitively. - -+'update':: -+ This command will try to update the local database. -+ + -+ It will terminate with a return code of zero if the database has been -+ successfully updated. 1 on error, 2 on invalid call and 3 if the -+ database was already the latest version. -+ -+'verify':: -+ Verifies the downloaded database. -+ - 'list-networks-by-as [--family=[ipv6|ipv4]] [--output-format FORMAT] ASN':: - Lists all networks which belong to this Autonomous System. - + -@@ -85,6 +97,12 @@ The 'location-query' command will normally exit with code zero. - If there has been a problem and the requested action could not be performed, - the exit code is unequal to zero. - -+== HOW IT WORKS -+The downloader checks a DNS record for the latest version of the database. -+It will then try to download a file with that version from a mirror server. -+If the downloaded file is outdated, the next mirror will be tried until we -+have found a file that is recent enough. -+ - == BUGS - Please report all bugs to the bugtracker at https://bugzilla.ipfire.org/. - diff --git a/src/patches/libloc-0.9.1-merge-location-exporter-into-location.patch b/src/patches/libloc-0.9.1-merge-location-exporter-into-location.patch deleted file mode 100644 index cda0cb8e69..0000000000 --- a/src/patches/libloc-0.9.1-merge-location-exporter-into-location.patch +++ /dev/null @@ -1,796 +0,0 @@ -commit 88ef7e9cd4b3a1a5662c7dc071bd7a44e1242cba -Author: Michael Tremer -Date: Wed Jun 3 18:36:28 2020 +0000 - - Merge location-exporter(8) into location(8) - - Signed-off-by: Michael Tremer - -diff --git a/Makefile.am b/Makefile.am -index 59870b1..9f520cc 100644 ---- a/Makefile.am -+++ b/Makefile.am -@@ -150,6 +150,7 @@ dist_pkgpython_PYTHON = \ - src/python/__init__.py \ - src/python/database.py \ - src/python/downloader.py \ -+ src/python/export.py \ - src/python/i18n.py \ - src/python/importer.py \ - src/python/logger.py -@@ -239,17 +240,14 @@ uninstall-perl: - - bin_SCRIPTS = \ - src/python/location \ -- src/python/location-exporter \ - src/python/location-importer - - EXTRA_DIST += \ - src/python/location.in \ -- src/python/location-exporter.in \ - src/python/location-importer.in - - CLEANFILES += \ - src/python/location \ -- src/python/location-exporter \ - src/python/location-importer - - # ------------------------------------------------------------------------------ -diff --git a/src/python/export.py b/src/python/export.py -new file mode 100644 -index 0000000..69fe964 ---- /dev/null -+++ b/src/python/export.py -@@ -0,0 +1,185 @@ -+#!/usr/bin/python3 -+############################################################################### -+# # -+# libloc - A library to determine the location of someone on the Internet # -+# # -+# Copyright (C) 2020 IPFire Development Team # -+# # -+# This library is free software; you can redistribute it and/or # -+# modify it under the terms of the GNU Lesser General Public # -+# License as published by the Free Software Foundation; either # -+# version 2.1 of the License, or (at your option) any later version. # -+# # -+# This library is distributed in the hope that it will be useful, # -+# but WITHOUT ANY WARRANTY; without even the implied warranty of # -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # -+# Lesser General Public License for more details. # -+# # -+############################################################################### -+ -+import io -+import ipaddress -+import logging -+import os -+import socket -+ -+# Initialise logging -+log = logging.getLogger("location.export") -+log.propagate = 1 -+ -+class OutputWriter(object): -+ suffix = "networks" -+ mode = "w" -+ -+ def __init__(self, f, prefix=None): -+ self.f, self.prefix = f, prefix -+ -+ # Immediately write the header -+ self._write_header() -+ -+ @classmethod -+ def open(cls, filename, **kwargs): -+ """ -+ Convenience function to open a file -+ """ -+ f = open(filename, cls.mode) -+ -+ return cls(f, **kwargs) -+ -+ def __repr__(self): -+ return "<%s f=%s>" % (self.__class__.__name__, self.f) -+ -+ def _write_header(self): -+ """ -+ The header of the file -+ """ -+ pass -+ -+ def _write_footer(self): -+ """ -+ The footer of the file -+ """ -+ pass -+ -+ def write(self, network): -+ self.f.write("%s\n" % network) -+ -+ def finish(self): -+ """ -+ Called when all data has been written -+ """ -+ self._write_footer() -+ -+ # Close the file -+ self.f.close() -+ -+ -+class IpsetOutputWriter(OutputWriter): -+ """ -+ For ipset -+ """ -+ suffix = "ipset" -+ -+ def _write_header(self): -+ self.f.write("create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.prefix) -+ -+ def write(self, network): -+ self.f.write("add %s %s\n" % (self.prefix, network)) -+ -+ -+class NftablesOutputWriter(OutputWriter): -+ """ -+ For nftables -+ """ -+ suffix = "set" -+ -+ def _write_header(self): -+ self.f.write("define %s = {\n" % self.prefix) -+ -+ def _write_footer(self): -+ self.f.write("}\n") -+ -+ def write(self, network): -+ self.f.write(" %s,\n" % network) -+ -+ -+class XTGeoIPOutputWriter(OutputWriter): -+ """ -+ Formats the output in that way, that it can be loaded by -+ the xt_geoip kernel module from xtables-addons. -+ """ -+ suffix = "iv" -+ mode = "wb" -+ -+ def write(self, network): -+ n = ipaddress.ip_network("%s" % network) -+ -+ for address in (n.network_address, n.broadcast_address): -+ bytes = socket.inet_pton( -+ socket.AF_INET6 if address.version == 6 else socket.AF_INET, -+ "%s" % address, -+ ) -+ -+ self.f.write(bytes) -+ -+ -+formats = { -+ "ipset" : IpsetOutputWriter, -+ "list" : OutputWriter, -+ "nftables" : NftablesOutputWriter, -+ "xt_geoip" : XTGeoIPOutputWriter, -+} -+ -+class Exporter(object): -+ def __init__(self, db, writer): -+ self.db, self.writer = db, writer -+ -+ def export(self, directory, families, countries, asns): -+ for family in families: -+ log.debug("Exporting family %s" % family) -+ -+ writers = {} -+ -+ # Create writers for countries -+ for country_code in countries: -+ filename = self._make_filename( -+ directory, prefix=country_code, suffix=self.writer.suffix, family=family, -+ ) -+ -+ writers[country_code] = self.writer.open(filename, prefix="CC_%s" % country_code) -+ -+ # Create writers for ASNs -+ for asn in asns: -+ filename = self._make_filename( -+ directory, "AS%s" % asn, suffix=self.writer.suffix, family=family, -+ ) -+ -+ writers[asn] = self.writer.open(filename, prefix="AS%s" % asn) -+ -+ # Get all networks that match the family -+ networks = self.db.search_networks(family=family) -+ -+ # Walk through all networks -+ for network in networks: -+ # Write matching countries -+ try: -+ writers[network.country_code].write(network) -+ except KeyError: -+ pass -+ -+ # Write matching ASNs -+ try: -+ writers[network.asn].write(network) -+ except KeyError: -+ pass -+ -+ # Write everything to the filesystem -+ for writer in writers.values(): -+ writer.finish() -+ -+ def _make_filename(self, directory, prefix, suffix, family): -+ filename = "%s.%s%s" % ( -+ prefix, suffix, "6" if family == socket.AF_INET6 else "4" -+ ) -+ -+ return os.path.join(directory, filename) -diff --git a/src/python/location-exporter.in b/src/python/location-exporter.in -deleted file mode 100644 -index d82f1d3..0000000 ---- a/src/python/location-exporter.in -+++ /dev/null -@@ -1,300 +0,0 @@ --#!/usr/bin/python3 --############################################################################### --# # --# libloc - A library to determine the location of someone on the Internet # --# # --# Copyright (C) 2019 IPFire Development Team # --# # --# This library is free software; you can redistribute it and/or # --# modify it under the terms of the GNU Lesser General Public # --# License as published by the Free Software Foundation; either # --# version 2.1 of the License, or (at your option) any later version. # --# # --# This library is distributed in the hope that it will be useful, # --# but WITHOUT ANY WARRANTY; without even the implied warranty of # --# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # --# Lesser General Public License for more details. # --# # --############################################################################### -- --import argparse --import io --import ipaddress --import logging --import os.path --import re --import socket --import sys -- --# Load our location module --import location --from location.i18n import _ -- --# Initialise logging --log = logging.getLogger("location.exporter") --log.propagate = 1 -- --class OutputWriter(object): -- suffix = "networks" -- -- def __init__(self, family, country_code=None, asn=None): -- self.family, self.country_code, self.asn = family, country_code, asn -- -- self.f = io.BytesIO() -- -- def write_out(self, directory): -- # Make the output filename -- filename = os.path.join( -- directory, self._make_filename(), -- ) -- -- with open(filename, "wb") as f: -- self._write_header(f) -- -- # Copy all data into the file -- f.write(self.f.getbuffer()) -- -- self._write_footer(f) -- -- def _make_filename(self): -- return "%s.%s%s" % ( -- self.country_code or "AS%s" % self.asn, -- self.suffix, -- "6" if self.family == socket.AF_INET6 else "4" -- ) -- -- @property -- def name(self): -- if self.country_code: -- return "CC_%s" % self.country_code -- -- if self.asn: -- return "AS%s" % self.asn -- -- def _write_header(self, f): -- """ -- The header of the file -- """ -- pass -- -- def _write_footer(self, f): -- """ -- The footer of the file -- """ -- pass -- -- def write(self, network): -- s = "%s\n" % network -- -- self.f.write(s.encode("ascii")) -- -- --class IpsetOutputWriter(OutputWriter): -- """ -- For ipset -- """ -- suffix = "ipset" -- -- def _write_header(self, f): -- h = "create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.name -- -- f.write(h.encode("ascii")) -- -- def write(self, network): -- s = "add %s %s\n" % (self.name, network) -- -- self.f.write(s.encode("ascii")) -- -- --class NftablesOutputWriter(OutputWriter): -- """ -- For nftables -- """ -- suffix = "set" -- -- def _write_header(self, f): -- h = "define %s = {\n" % self.name -- -- f.write(h.encode("ascii")) -- -- def _write_footer(self, f): -- f.write(b"}") -- -- def write(self, network): -- s = " %s,\n" % network -- -- self.f.write(s.encode("ascii")) -- -- --class XTGeoIPOutputWriter(OutputWriter): -- """ -- Formats the output in that way, that it can be loaded by -- the xt_geoip kernel module from xtables-addons. -- """ -- suffix = "iv" -- -- def write(self, network): -- n = ipaddress.ip_network("%s" % network) -- -- for address in (n.network_address, n.broadcast_address): -- bytes = socket.inet_pton( -- socket.AF_INET6 if address.version == 6 else socket.AF_INET, -- "%s" % address, -- ) -- -- self.f.write(bytes) -- -- --class Exporter(object): -- def __init__(self, db, writer): -- self.db = db -- self.writer = writer -- -- def export(self, directory, families, countries, asns): -- for family in families: -- log.debug("Exporting family %s" % family) -- -- writers = {} -- -- # Create writers for countries -- for country_code in countries: -- writers[country_code] = self.writer(family, country_code=country_code) -- -- # Create writers for ASNs -- for asn in asns: -- writers[asn] = self.writer(family, asn=asn) -- -- # Get all networks that match the family -- networks = self.db.search_networks(family=family) -- -- # Walk through all networks -- for network in networks: -- # Write matching countries -- if network.country_code in countries: -- writers[network.country_code].write(network) -- -- # Write matching ASNs -- if network.asn in asns: -- writers[network.asn].write(network) -- -- # Write everything to the filesystem -- for writer in writers.values(): -- writer.write_out(directory) -- -- --class CLI(object): -- output_formats = { -- "ipset" : IpsetOutputWriter, -- "list" : OutputWriter, -- "nftables" : NftablesOutputWriter, -- "xt_geoip" : XTGeoIPOutputWriter, -- } -- -- def parse_cli(self): -- parser = argparse.ArgumentParser( -- description=_("Location Exporter Command Line Interface"), -- ) -- -- # Global configuration flags -- parser.add_argument("--debug", action="store_true", -- help=_("Enable debug output")) -- parser.add_argument("--quiet", action="store_true", -- help=_("Enable quiet mode")) -- -- # version -- parser.add_argument("--version", action="version", -- version="%(prog)s @VERSION@") -- -- # database -- parser.add_argument("--database", "-d", -- default="@databasedir@/database.db", help=_("Path to database"), -- ) -- -- # format -- parser.add_argument("--format", help=_("Output format"), -- default="list", choices=self.output_formats.keys()) -- -- # directory -- parser.add_argument("--directory", help=_("Output directory"), required=True) -- -- # family -- parser.add_argument("--family", help=_("Specify address family"), choices=("ipv6", "ipv4")) -- -- # Countries and Autonomous Systems -- parser.add_argument("objects", nargs="+") -- -- args = parser.parse_args() -- -- # Configure logging -- if args.debug: -- location.logger.set_level(logging.DEBUG) -- elif args.quiet: -- location.logger.set_level(logging.WARNING) -- -- return args -- -- def run(self): -- # Parse command line arguments -- args = self.parse_cli() -- -- # Call function -- ret = self.handle_export(args) -- -- # Return with exit code -- if ret: -- sys.exit(ret) -- -- # Otherwise just exit -- sys.exit(0) -- -- def handle_export(self, ns): -- countries, asns = [], [] -- -- # Translate family -- if ns.family == "ipv6": -- families = [ socket.AF_INET6 ] -- elif ns.family == "ipv4": -- families = [ socket.AF_INET ] -- else: -- families = [ socket.AF_INET6, socket.AF_INET ] -- -- for object in ns.objects: -- m = re.match("^AS(\d+)$", object) -- if m: -- object = int(m.group(1)) -- -- asns.append(object) -- -- elif location.country_code_is_valid(object) \ -- or object in ("A1", "A2", "A3"): -- countries.append(object) -- -- else: -- log.warning("Invalid argument: %s" % object) -- continue -- -- if not countries and not asns: -- log.error("Nothing to export") -- return 2 -- -- # Open the database -- try: -- db = location.Database(ns.database) -- except FileNotFoundError as e: -- log.error("Count not open database: %s" % ns.database) -- return 1 -- -- # Select the output format -- writer = self.output_formats.get(ns.format) -- assert writer -- -- e = Exporter(db, writer) -- e.export(ns.directory, countries=countries, asns=asns, families=families) -- -- --def main(): -- # Run the command line interface -- c = CLI() -- c.run() -- --main() -diff --git a/src/python/location.in b/src/python/location.in -index 10618e2..7614cae 100644 ---- a/src/python/location.in -+++ b/src/python/location.in -@@ -22,6 +22,7 @@ import datetime - import ipaddress - import logging - import os -+import re - import shutil - import socket - import sys -@@ -30,6 +31,8 @@ import time - # Load our location module - import location - import location.downloader -+import location.export -+ - from location.i18n import _ - - # Setup logging -@@ -37,88 +40,7 @@ log = logging.getLogger("location") - - # Output formatters - --class OutputFormatter(object): -- def __init__(self, ns): -- self.ns = ns -- -- def __enter__(self): -- # Open the output -- self.open() -- -- return self -- -- def __exit__(self, type, value, tb): -- if tb is None: -- self.close() -- -- @property -- def name(self): -- if "country_code" in self.ns: -- return "networks_country_%s" % self.ns.country_code[0] -- -- elif "asn" in self.ns: -- return "networks_AS%s" % self.ns.asn[0] -- -- def open(self): -- pass -- -- def close(self): -- pass -- -- def network(self, network): -- print(network) -- -- --class IpsetOutputFormatter(OutputFormatter): -- """ -- For nftables -- """ -- def open(self): -- print("create %s hash:net family inet hashsize 1024 maxelem 65536" % self.name) -- -- def network(self, network): -- print("add %s %s" % (self.name, network)) -- -- --class NftablesOutputFormatter(OutputFormatter): -- """ -- For nftables -- """ -- def open(self): -- print("define %s = {" % self.name) -- -- def close(self): -- print("}") -- -- def network(self, network): -- print(" %s," % network) -- -- --class XTGeoIPOutputFormatter(OutputFormatter): -- """ -- Formats the output in that way, that it can be loaded by -- the xt_geoip kernel module from xtables-addons. -- """ -- def network(self, network): -- n = ipaddress.ip_network("%s" % network) -- -- for address in (n.network_address, n.broadcast_address): -- bytes = socket.inet_pton( -- socket.AF_INET6 if address.version == 6 else socket.AF_INET, -- "%s" % address, -- ) -- -- os.write(1, bytes) -- -- - class CLI(object): -- output_formats = { -- "ipset" : IpsetOutputFormatter, -- "list" : OutputFormatter, -- "nftables" : NftablesOutputFormatter, -- "xt_geoip" : XTGeoIPOutputFormatter, -- } -- - def parse_cli(self): - parser = argparse.ArgumentParser( - description=_("Location Database Command Line Interface"), -@@ -193,8 +115,8 @@ class CLI(object): - ) - list_networks_by_as.add_argument("asn", nargs=1, type=int) - list_networks_by_as.add_argument("--family", choices=("ipv6", "ipv4")) -- list_networks_by_as.add_argument("--output-format", -- choices=self.output_formats.keys(), default="list") -+ list_networks_by_as.add_argument("--format", -+ choices=location.export.formats.keys(), default="list") - list_networks_by_as.set_defaults(func=self.handle_list_networks_by_as) - - # List all networks in a country -@@ -203,8 +125,8 @@ class CLI(object): - ) - list_networks_by_cc.add_argument("country_code", nargs=1) - list_networks_by_cc.add_argument("--family", choices=("ipv6", "ipv4")) -- list_networks_by_cc.add_argument("--output-format", -- choices=self.output_formats.keys(), default="list") -+ list_networks_by_cc.add_argument("--format", -+ choices=location.export.formats.keys(), default="list") - list_networks_by_cc.set_defaults(func=self.handle_list_networks_by_cc) - - # List all networks with flags -@@ -221,10 +143,23 @@ class CLI(object): - action="store_true", help=_("Anycasts"), - ) - list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4")) -- list_networks_by_flags.add_argument("--output-format", -- choices=self.output_formats.keys(), default="list") -+ list_networks_by_flags.add_argument("--format", -+ choices=location.export.formats.keys(), default="list") - list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags) - -+ # Export -+ export = subparsers.add_parser("export", -+ help=_("Exports data in many formats to load it into packet filters"), -+ ) -+ export.add_argument("--format", help=_("Output format"), -+ choices=location.export.formats.keys(), default="list") -+ export.add_argument("--directory", help=_("Output directory"), required=True) -+ export.add_argument("--family", -+ help=_("Specify address family"), choices=("ipv6", "ipv4"), -+ ) -+ export.add_argument("objects", nargs="+", help=_("List country codes or ASNs to export")) -+ export.set_defaults(func=self.handle_export) -+ - args = parser.parse_args() - - # Configure logging -@@ -494,25 +429,36 @@ class CLI(object): - - def __get_output_formatter(self, ns): - try: -- cls = self.output_formats[ns.output_format] -+ cls = location.export.formats[ns.format] - except KeyError: -- cls = OutputFormatter -+ cls = location.export.OutputFormatter - -- return cls(ns) -+ return cls - - def handle_list_networks_by_as(self, db, ns): -- with self.__get_output_formatter(ns) as f: -- for asn in ns.asn: -- # Print all matching networks -- for n in db.search_networks(asn=asn, family=ns.family): -- f.network(n) -+ writer = self.__get_output_formatter(ns) -+ -+ for asn in ns.asn: -+ f = writer(sys.stdout, prefix="AS%s" % asn) -+ -+ # Print all matching networks -+ for n in db.search_networks(asn=asn, family=ns.family): -+ f.write(n) -+ -+ f.finish() - - def handle_list_networks_by_cc(self, db, ns): -- with self.__get_output_formatter(ns) as f: -- for country_code in ns.country_code: -- # Print all matching networks -- for n in db.search_networks(country_code=country_code, family=ns.family): -- f.network(n) -+ writer = self.__get_output_formatter(ns) -+ -+ for country_code in ns.country_code: -+ # Open standard output -+ f = writer(sys.stdout, prefix=country_code) -+ -+ # Print all matching networks -+ for n in db.search_networks(country_code=country_code, family=ns.family): -+ f.write(n) -+ -+ f.finish() - - def handle_list_networks_by_flags(self, db, ns): - flags = 0 -@@ -529,9 +475,49 @@ class CLI(object): - if not flags: - raise ValueError(_("You must at least pass one flag")) - -- with self.__get_output_formatter(ns) as f: -- for n in db.search_networks(flags=flags, family=ns.family): -- f.network(n) -+ writer = self.__get_output_formatter(ns) -+ f = writer(sys.stdout, prefix="custom") -+ -+ for n in db.search_networks(flags=flags, family=ns.family): -+ f.write(n) -+ -+ f.finish() -+ -+ def handle_export(self, db, ns): -+ countries, asns = [], [] -+ -+ # Translate family -+ if ns.family == "ipv6": -+ families = [ socket.AF_INET6 ] -+ elif ns.family == "ipv4": -+ families = [ socket.AF_INET ] -+ else: -+ families = [ socket.AF_INET6, socket.AF_INET ] -+ -+ for object in ns.objects: -+ m = re.match("^AS(\d+)$", object) -+ if m: -+ object = int(m.group(1)) -+ -+ asns.append(object) -+ -+ elif location.country_code_is_valid(object) \ -+ or object in ("A1", "A2", "A3"): -+ countries.append(object) -+ -+ else: -+ log.warning("Invalid argument: %s" % object) -+ continue -+ -+ if not countries and not asns: -+ log.error("Nothing to export") -+ return 2 -+ -+ # Select the output format -+ writer = self.__get_output_formatter(ns) -+ -+ e = location.export.Exporter(db, writer) -+ e.export(ns.directory, countries=countries, asns=asns, families=families) - - - def main(): diff --git a/src/patches/libloc-0.9.1-move-location-downloader-functionality-into-location-query.patch b/src/patches/libloc-0.9.1-move-location-downloader-functionality-into-location-query.patch deleted file mode 100644 index c14093fa53..0000000000 --- a/src/patches/libloc-0.9.1-move-location-downloader-functionality-into-location-query.patch +++ /dev/null @@ -1,383 +0,0 @@ -commit a6f1e3463d4c2085c203ad58072d7a154b663904 -Author: Michael Tremer -Date: Wed Jun 3 17:06:13 2020 +0000 - - Move location-downloader functionality into location-query - - The commands are very long and confusion. Hence we merge this - all into one command. - - Signed-off-by: Michael Tremer - -diff --git a/Makefile.am b/Makefile.am -index 31869e0..c0b1300 100644 ---- a/Makefile.am -+++ b/Makefile.am -@@ -146,6 +146,7 @@ CLEANFILES += \ - dist_pkgpython_PYTHON = \ - src/python/__init__.py \ - src/python/database.py \ -+ src/python/downloader.py \ - src/python/i18n.py \ - src/python/importer.py \ - src/python/logger.py -@@ -234,19 +235,16 @@ uninstall-perl: - $(DESTDIR)/$(prefix)/man/man3/Location.3pm - - bin_SCRIPTS = \ -- src/python/location-downloader \ - src/python/location-exporter \ - src/python/location-importer \ - src/python/location-query - - EXTRA_DIST += \ -- src/python/location-downloader.in \ - src/python/location-exporter.in \ - src/python/location-importer.in \ - src/python/location-query.in - - CLEANFILES += \ -- src/python/location-downloader \ - src/python/location-exporter \ - src/python/location-importer \ - src/python/location-query -diff --git a/src/python/location-downloader.in b/src/python/downloader.py -similarity index 60% -rename from src/python/location-downloader.in -rename to src/python/downloader.py -index bf0d682..c9e6e00 100644 ---- a/src/python/location-downloader.in -+++ b/src/python/downloader.py -@@ -3,7 +3,7 @@ - # # - # libloc - A library to determine the location of someone on the Internet # - # # --# Copyright (C) 2019 IPFire Development Team # -+# Copyright (C) 2020 IPFire Development Team # - # # - # This library is free software; you can redistribute it and/or # - # modify it under the terms of the GNU Lesser General Public # -@@ -17,24 +17,18 @@ - # # - ############################################################################### - --import argparse --import datetime - import logging - import lzma - import os - import random --import shutil - import stat --import sys - import tempfile - import time - import urllib.error - import urllib.parse - import urllib.request - --# Load our location module --import location --from location.i18n import _ -+from _location import Database, DATABASE_VERSION_LATEST - - DATABASE_FILENAME = "location.db.xz" - MIRRORS = ( -@@ -46,9 +40,11 @@ log = logging.getLogger("location.downloader") - log.propagate = 1 - - class Downloader(object): -- def __init__(self, version, mirrors): -+ def __init__(self, version=DATABASE_VERSION_LATEST, mirrors=None): - self.version = version -- self.mirrors = list(mirrors) -+ -+ # Set mirrors or use defaults -+ self.mirrors = list(mirrors or MIRRORS) - - # Randomize mirrors - random.shuffle(self.mirrors) -@@ -117,9 +113,10 @@ class Downloader(object): - - return res - -- def download(self, url, public_key, timestamp=None, tmpdir=None, **kwargs): -- headers = {} -+ def download(self, public_key, timestamp=None, tmpdir=None, **kwargs): -+ url = "%s/%s" % (self.version, DATABASE_FILENAME) - -+ headers = {} - if timestamp: - headers["If-Modified-Since"] = timestamp.strftime( - "%a, %d %b %Y %H:%M:%S GMT", -@@ -191,7 +188,7 @@ class Downloader(object): - """ - log.debug("Opening downloaded database at %s" % f.name) - -- db = location.Database(f.name) -+ db = Database(f.name) - - # Database is not recent - if timestamp and db.created_at < timestamp.timestamp(): -@@ -208,141 +205,3 @@ class Downloader(object): - return False - - return True -- -- --class CLI(object): -- def __init__(self): -- # Which version are we downloading? -- self.version = location.DATABASE_VERSION_LATEST -- -- self.downloader = Downloader(version=self.version, mirrors=MIRRORS) -- -- def parse_cli(self): -- parser = argparse.ArgumentParser( -- description=_("Location Downloader Command Line Interface"), -- ) -- subparsers = parser.add_subparsers() -- -- # Global configuration flags -- parser.add_argument("--debug", action="store_true", -- help=_("Enable debug output")) -- parser.add_argument("--quiet", action="store_true", -- help=_("Enable quiet mode")) -- -- # version -- parser.add_argument("--version", action="version", -- version="%(prog)s @VERSION@") -- -- # database -- parser.add_argument("--database", "-d", -- default="@databasedir@/database.db", help=_("Path to database"), -- ) -- -- # public key -- parser.add_argument("--public-key", "-k", -- default="@databasedir@/signing-key.pem", help=_("Public Signing Key"), -- ) -- -- # Update -- update = subparsers.add_parser("update", help=_("Update database")) -- update.set_defaults(func=self.handle_update) -- -- # Verify -- verify = subparsers.add_parser("verify", -- help=_("Verify the downloaded database")) -- verify.set_defaults(func=self.handle_verify) -- -- args = parser.parse_args() -- -- # Configure logging -- if args.debug: -- location.logger.set_level(logging.DEBUG) -- elif args.quiet: -- location.logger.set_level(logging.WARNING) -- -- # Print usage if no action was given -- if not "func" in args: -- parser.print_usage() -- sys.exit(2) -- -- return args -- -- def run(self): -- # Parse command line arguments -- args = self.parse_cli() -- -- # Call function -- ret = args.func(args) -- -- # Return with exit code -- if ret: -- sys.exit(ret) -- -- # Otherwise just exit -- sys.exit(0) -- -- def handle_update(self, ns): -- # Fetch the timestamp we need from DNS -- t = location.discover_latest_version(self.version) -- -- # Parse timestamp into datetime format -- timestamp = datetime.datetime.fromtimestamp(t) if t else None -- -- # Open database -- try: -- db = location.Database(ns.database) -- -- # Check if we are already on the latest version -- if timestamp and db.created_at >= timestamp.timestamp(): -- log.info("Already on the latest version") -- return -- -- except FileNotFoundError as e: -- db = None -- -- # Download the database into the correct directory -- tmpdir = os.path.dirname(ns.database) -- -- # Try downloading a new database -- try: -- t = self.downloader.download("%s/%s" % (self.version, DATABASE_FILENAME), -- public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir) -- -- # If no file could be downloaded, log a message -- except FileNotFoundError as e: -- log.error("Could not download a new database") -- return 1 -- -- # If we have not received a new file, there is nothing to do -- if not t: -- return 3 -- -- # Move temporary file to destination -- shutil.move(t.name, ns.database) -- -- return 0 -- -- def handle_verify(self, ns): -- try: -- db = location.Database(ns.database) -- except FileNotFoundError as e: -- log.error("%s: %s" % (ns.database, e)) -- return 127 -- -- # Verify the database -- with open(ns.public_key, "r") as f: -- if not db.verify(f): -- log.error("Could not verify database") -- return 1 -- -- # Success -- log.debug("Database successfully verified") -- return 0 -- -- --def main(): -- # Run the command line interface -- c = CLI() -- c.run() -- --main() -diff --git a/src/python/location-query.in b/src/python/location-query.in -index dfdff8c..0291786 100644 ---- a/src/python/location-query.in -+++ b/src/python/location-query.in -@@ -18,16 +18,23 @@ - ############################################################################### - - import argparse -+import datetime - import ipaddress -+import logging - import os -+import shutil - import socket - import sys - import time - - # Load our location module - import location -+import location.downloader - from location.i18n import _ - -+# Setup logging -+log = logging.getLogger("location") -+ - # Output formatters - - class OutputFormatter(object): -@@ -157,6 +164,15 @@ class CLI(object): - dump.add_argument("output", nargs="?", type=argparse.FileType("w")) - dump.set_defaults(func=self.handle_dump) - -+ # Update -+ update = subparsers.add_parser("update", help=_("Update database")) -+ update.set_defaults(func=self.handle_update) -+ -+ # Verify -+ verify = subparsers.add_parser("verify", -+ help=_("Verify the downloaded database")) -+ verify.set_defaults(func=self.handle_verify) -+ - # Get AS - get_as = subparsers.add_parser("get-as", - help=_("Get information about one or multiple Autonomous Systems"), -@@ -423,6 +439,59 @@ class CLI(object): - for a in db.search_as(query): - print(a) - -+ def handle_update(self, db, ns): -+ # Fetch the timestamp we need from DNS -+ t = location.discover_latest_version() -+ -+ # Parse timestamp into datetime format -+ timestamp = datetime.datetime.fromtimestamp(t) if t else None -+ -+ # Check the version of the local database -+ if db and timestamp and db.created_at >= timestamp.timestamp(): -+ log.info("Already on the latest version") -+ return -+ -+ # Download the database into the correct directory -+ tmpdir = os.path.dirname(ns.database) -+ -+ # Create a downloader -+ d = location.downloader.Downloader() -+ -+ # Try downloading a new database -+ try: -+ t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir) -+ -+ # If no file could be downloaded, log a message -+ except FileNotFoundError as e: -+ log.error("Could not download a new database") -+ return 1 -+ -+ # If we have not received a new file, there is nothing to do -+ if not t: -+ return 3 -+ -+ # Move temporary file to destination -+ shutil.move(t.name, ns.database) -+ -+ return 0 -+ -+ def handle_verify(self, ns): -+ try: -+ db = location.Database(ns.database) -+ except FileNotFoundError as e: -+ log.error("%s: %s" % (ns.database, e)) -+ return 127 -+ -+ # Verify the database -+ with open(ns.public_key, "r") as f: -+ if not db.verify(f): -+ log.error("Could not verify database") -+ return 1 -+ -+ # Success -+ log.debug("Database successfully verified") -+ return 0 -+ - def __get_output_formatter(self, ns): - try: - cls = self.output_formats[ns.output_format] -diff --git a/src/python/locationmodule.c b/src/python/locationmodule.c -index a04cab7..5b72be9 100644 ---- a/src/python/locationmodule.c -+++ b/src/python/locationmodule.c -@@ -50,9 +50,9 @@ static PyObject* set_log_level(PyObject* m, PyObject* args) { - } - - static PyObject* discover_latest_version(PyObject* m, PyObject* args) { -- unsigned int version = 0; -+ unsigned int version = LOC_DATABASE_VERSION_LATEST; - -- if (!PyArg_ParseTuple(args, "i", &version)) -+ if (!PyArg_ParseTuple(args, "|i", &version)) - return NULL; - - time_t t = 0; diff --git a/src/patches/libloc-0.9.1-remove-accidently-commited-hacks-for-debian.patch b/src/patches/libloc-0.9.1-remove-accidently-commited-hacks-for-debian.patch deleted file mode 100644 index f0df20a6e0..0000000000 --- a/src/patches/libloc-0.9.1-remove-accidently-commited-hacks-for-debian.patch +++ /dev/null @@ -1,22 +0,0 @@ -commit 6bfde1447d237d2a345b99677c5b74e54cbd5739 -Author: Michael Tremer -Date: Thu Jun 4 10:37:50 2020 +0000 - - Makefile: Remove accidentially committed hacks for Debian - - Signed-off-by: Michael Tremer - -diff --git a/Makefile.am b/Makefile.am -index ef57551..c75839c 100644 ---- a/Makefile.am -+++ b/Makefile.am -@@ -37,9 +37,6 @@ LIBLOC_CURRENT=0 - LIBLOC_REVISION=0 - LIBLOC_AGE=0 - --pythondir = $(prefix)/lib/python3/dist-packages --pyexecdir = $(prefix)/lib/python$(PYTHON_VERSION)/lib-dynload -- - DISTCHECK_CONFIGURE_FLAGS = \ - --with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir) - diff --git a/src/patches/libloc-0.9.1-remove-python-path-overrides-for-debian.patch b/src/patches/libloc-0.9.1-remove-python-path-overrides-for-debian.patch deleted file mode 100644 index 0656a030c2..0000000000 --- a/src/patches/libloc-0.9.1-remove-python-path-overrides-for-debian.patch +++ /dev/null @@ -1,37 +0,0 @@ -commit 9df8db2ae6268b0901961625fd27b4dde1ed451f -Author: Michael Tremer -Date: Mon Jun 1 18:23:50 2020 +0000 - - Makefile: Remove Python path overrides for Debian - - Signed-off-by: Michael Tremer - -diff --git a/Makefile.am b/Makefile.am -index 570ec3a..31869e0 100644 ---- a/Makefile.am -+++ b/Makefile.am -@@ -54,10 +54,6 @@ SED_PROCESS = \ - databasedir = $(localstatedir)/lib/location - pkgconfigdir = $(libdir)/pkgconfig - --# XXX hardcode path for Debian --pythondir = $(prefix)/lib/python3/dist-packages --pyexecdir = $(prefix)/lib/python$(PYTHON_VERSION)/lib-dynload -- - # Overwrite Python path - pkgpythondir = $(pythondir)/location - -diff --git a/debian/libloc.install b/debian/libloc.install -index eb3b49a..30bbeca 100644 ---- a/debian/libloc.install -+++ b/debian/libloc.install -@@ -2,8 +2,7 @@ usr/bin/location-downloader - usr/bin/location-exporter - usr/bin/location-query - usr/lib/*/libloc.so.* --usr/lib/python3*/dist-packages --usr/lib/python3*/lib-dynload/*.so -+usr/lib/python3*/site-packages - src/systemd/*.service /lib/systemd/system/ - src/systemd/*.timer /lib/systemd/system/ - var/lib/location/signing-key.pem diff --git a/src/patches/libloc-0.9.1-rename-location-query-to-location.patch b/src/patches/libloc-0.9.1-rename-location-query-to-location.patch deleted file mode 100644 index 4a86013afd..0000000000 --- a/src/patches/libloc-0.9.1-rename-location-query-to-location.patch +++ /dev/null @@ -1,111 +0,0 @@ -commit 1d237439676e8b9ee10a6dde2c64f5ba3a057210 -Author: Michael Tremer -Date: Wed Jun 3 17:21:31 2020 +0000 - - Rename location-query(8) to location(8) - - Signed-off-by: Michael Tremer - -diff --git a/Makefile.am b/Makefile.am -index bf204d8..59870b1 100644 ---- a/Makefile.am -+++ b/Makefile.am -@@ -238,19 +238,19 @@ uninstall-perl: - $(DESTDIR)/$(prefix)/man/man3/Location.3pm - - bin_SCRIPTS = \ -+ src/python/location \ - src/python/location-exporter \ -- src/python/location-importer \ -- src/python/location-query -+ src/python/location-importer - - EXTRA_DIST += \ -+ src/python/location.in \ - src/python/location-exporter.in \ -- src/python/location-importer.in \ -- src/python/location-query.in -+ src/python/location-importer.in - - CLEANFILES += \ -+ src/python/location \ - src/python/location-exporter \ -- src/python/location-importer \ -- src/python/location-query -+ src/python/location-importer - - # ------------------------------------------------------------------------------ - -@@ -374,7 +374,7 @@ src_test_signature_LDADD = \ - # ------------------------------------------------------------------------------ - - MANPAGES = \ -- man/location-query.8 -+ man/location.8 - - MANPAGES_TXT = $(patsubst %.8,%.txt,$(MANPAGES)) - MANPAGES_HTML = $(patsubst %.txt,%.html,$(MANPAGES_TXT)) -diff --git a/man/location-query.txt b/man/location.txt -similarity index 84% -rename from man/location-query.txt -rename to man/location.txt -index acb43cd..672c2b2 100644 ---- a/man/location-query.txt -+++ b/man/location.txt -@@ -1,21 +1,21 @@ --= location-query(8) -+= location(8) - - == NAME --location-query - Query the location database -+location - Query the location database - - == SYNOPSIS - [verse] --`location-query lookup ADDRESS [ADDRESS...]` --`location-query get-as ASN [ASN...]` --`location-query search-as STRING` --`location-query update` --`location-query verify` --`location-query list-networks-by-as ASN` --`location-query list-networks-by-cc COUNTRY_CODE` --`location-query list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]` -+`location lookup ADDRESS [ADDRESS...]` -+`location get-as ASN [ASN...]` -+`location search-as STRING` -+`location update` -+`location verify` -+`location list-networks-by-as ASN` -+`location list-networks-by-cc COUNTRY_CODE` -+`location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]` - - == DESCRIPTION --The `location-query` retrieves information from the location database. -+`location` retrieves information from the location database. - This data can be used to determine someone's location on the Internet - and for building firewall rulesets to block access from certain ASes - or countries. -@@ -93,7 +93,7 @@ or countries. - Shows the program's version and exists. - - == EXIT CODES --The 'location-query' command will normally exit with code zero. -+The 'location' command will normally exit with code zero. - If there has been a problem and the requested action could not be performed, - the exit code is unequal to zero. - -diff --git a/src/python/location-query.in b/src/python/location.in -similarity index 99% -rename from src/python/location-query.in -rename to src/python/location.in -index 0291786..10618e2 100644 ---- a/src/python/location-query.in -+++ b/src/python/location.in -@@ -248,7 +248,7 @@ class CLI(object): - try: - db = location.Database(args.database) - except FileNotFoundError as e: -- sys.stderr.write("location-query: Could not open database %s: %s\n" \ -+ sys.stderr.write("location: Could not open database %s: %s\n" \ - % (args.database, e)) - sys.exit(1)