# Add upstream patches.
cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-location-downloader-do-not-change-content-of-open-database.patch
cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-remove-python-path-overrides-for-debian.patch
+ cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-location-query-require-at-least-one-flag.patch
+ cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-location-exporter-do-not-mistake-country-as-for-an-as-number.patch
+ cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-location-exporter-warn-but-do-not-fail-on-invalid-input.patch
+ cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-move-location-downloader-functionality-into-location-query.patch
+ cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-merge-location-downloader-manpage-into-location-query.patch
+ cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-downloader-rename-user-agent-to-location.patch
+ cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-rename-location-query-to-location.patch
+ cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-merge-location-exporter-into-location.patch
+ cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-remove-accidently-commited-hacks-for-debian.patch
+ cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-add-option-to-iterate-over-all-contries-and-print-them.patch
+ cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-export-all-countries-by-default.patch
+ cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-export-flagged-networks-with-their-faked-country-names-too.patch
+ cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-database-fix-brocken-search-for-networks-with-flags.patch
+ cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-adjust-format-to-print-ASes.patch
cd $(DIR_APP) && ./autogen.sh
cd $(DIR_APP) && ./configure \
--- /dev/null
+commit fa9a3663cb2dfb2490da43f6967f1a3a2948fc8a
+Author: Michael Tremer <michael.tremer@ipfire.org>
+Date: Fri Jun 5 09:41:28 2020 +0000
+
+ Add option to iterate over all countries and print them to the console
+
+ Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+
+diff --git a/man/location.txt b/man/location.txt
+index 672c2b2..0d70e0b 100644
+--- a/man/location.txt
++++ b/man/location.txt
+@@ -13,6 +13,7 @@ location - Query the location database
+ `location list-networks-by-as ASN`
+ `location list-networks-by-cc COUNTRY_CODE`
+ `location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]`
++`location list-countries [--show-name] [--show-continent]`
+
+ == DESCRIPTION
+ `location` retrieves information from the location database.
+@@ -86,6 +87,12 @@ or countries.
+ +
+ See above for usage of the '--family' and '--output-format' parameters.
+
++'list-countries [--show-name] [--show-continent]'::
++ Lists all countries known to the database.
++ +
++ With the optional parameters '--show-name' and '--show-continent', the name and
++ continent code will be printed, too.
++
+ '--help'::
+ Shows a short help text on using this program.
+
+diff --git a/src/database.c b/src/database.c
+index d919278..8e6c5ab 100644
+--- a/src/database.c
++++ b/src/database.c
+@@ -107,6 +107,9 @@ struct loc_database_enumerator {
+ // Index of the AS we are looking at
+ unsigned int as_index;
+
++ // Index of the country we are looking at
++ unsigned int country_index;
++
+ // Network state
+ struct in6_addr network_address;
+ struct loc_node_stack network_stack[MAX_STACK_DEPTH];
+@@ -1219,3 +1222,30 @@ LOC_EXPORT int loc_database_enumerator_next_network(
+
+ return 0;
+ }
++
++LOC_EXPORT int loc_database_enumerator_next_country(
++ struct loc_database_enumerator* enumerator, struct loc_country** country) {
++ *country = NULL;
++
++ // Do not do anything if not in country mode
++ if (enumerator->mode != LOC_DB_ENUMERATE_COUNTRIES)
++ return 0;
++
++ struct loc_database* db = enumerator->db;
++
++ while (enumerator->country_index < db->countries_count) {
++ // Fetch the next country
++ int r = loc_database_fetch_country(db, country, enumerator->country_index++);
++ if (r)
++ return r;
++
++ // We do not filter here, so it always is a match
++ return 0;
++ }
++
++ // Reset the index
++ enumerator->country_index = 0;
++
++ // We have searched through all of them
++ return 0;
++}
+diff --git a/src/libloc.sym b/src/libloc.sym
+index e9e8549..9a1e6f0 100644
+--- a/src/libloc.sym
++++ b/src/libloc.sym
+@@ -68,6 +68,7 @@ global:
+ # Database Enumerator
+ loc_database_enumerator_new;
+ loc_database_enumerator_next_as;
++ loc_database_enumerator_next_country;
+ loc_database_enumerator_next_network;
+ loc_database_enumerator_ref;
+ loc_database_enumerator_set_asn;
+diff --git a/src/loc/database.h b/src/loc/database.h
+index ab9ef72..43173dd 100644
+--- a/src/loc/database.h
++++ b/src/loc/database.h
+@@ -50,8 +50,9 @@ int loc_database_get_country(struct loc_database* db,
+ struct loc_country** country, const char* code);
+
+ enum loc_database_enumerator_mode {
+- LOC_DB_ENUMERATE_NETWORKS = 1,
+- LOC_DB_ENUMERATE_ASES = 2,
++ LOC_DB_ENUMERATE_NETWORKS = 1,
++ LOC_DB_ENUMERATE_ASES = 2,
++ LOC_DB_ENUMERATE_COUNTRIES = 3,
+ };
+
+ struct loc_database_enumerator;
+@@ -69,5 +70,7 @@ int loc_database_enumerator_next_as(
+ struct loc_database_enumerator* enumerator, struct loc_as** as);
+ int loc_database_enumerator_next_network(
+ struct loc_database_enumerator* enumerator, struct loc_network** network);
++int loc_database_enumerator_next_country(
++ struct loc_database_enumerator* enumerator, struct loc_country** country);
+
+ #endif
+diff --git a/src/python/database.c b/src/python/database.c
+index 581ed5b..1013a58 100644
+--- a/src/python/database.c
++++ b/src/python/database.c
+@@ -316,6 +316,10 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
+ return obj;
+ }
+
++static PyObject* Database_countries(DatabaseObject* self) {
++ return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES);
++}
++
+ static struct PyMethodDef Database_methods[] = {
+ {
+ "get_as",
+@@ -364,6 +368,13 @@ static struct PyGetSetDef Database_getsetters[] = {
+ NULL,
+ NULL,
+ },
++ {
++ "countries",
++ (getter)Database_countries,
++ NULL,
++ NULL,
++ NULL,
++ },
+ {
+ "created_at",
+ (getter)Database_get_created_at,
+@@ -462,6 +473,22 @@ static PyObject* DatabaseEnumerator_next(DatabaseEnumeratorObject* self) {
+ return obj;
+ }
+
++ // Enumerate all countries
++ struct loc_country* country = NULL;
++
++ r = loc_database_enumerator_next_country(self->enumerator, &country);
++ if (r) {
++ PyErr_SetFromErrno(PyExc_ValueError);
++ return NULL;
++ }
++
++ if (country) {
++ PyObject* obj = new_country(&CountryType, country);
++ loc_country_unref(country);
++
++ return obj;
++ }
++
+ // Nothing found, that means the end
+ PyErr_SetNone(PyExc_StopIteration);
+ return NULL;
+diff --git a/src/python/location.in b/src/python/location.in
+index 7614cae..5c1effd 100644
+--- a/src/python/location.in
++++ b/src/python/location.in
+@@ -147,6 +147,18 @@ class CLI(object):
+ choices=location.export.formats.keys(), default="list")
+ list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags)
+
++ # List countries
++ list_countries = subparsers.add_parser("list-countries",
++ help=_("Lists all countries"),
++ )
++ list_countries.add_argument("--show-name",
++ action="store_true", help=_("Show the name of the country"),
++ )
++ list_countries.add_argument("--show-continent",
++ action="store_true", help=_("Show the continent"),
++ )
++ list_countries.set_defaults(func=self.handle_list_countries)
++
+ # Export
+ export = subparsers.add_parser("export",
+ help=_("Exports data in many formats to load it into packet filters"),
+@@ -435,6 +447,24 @@ class CLI(object):
+
+ return cls
+
++ def handle_list_countries(self, db, ns):
++ for country in db.countries:
++ line = [
++ country.code,
++ ]
++
++ if ns.show_continent:
++ line.append(country.continent_code)
++
++ if ns.show_name:
++ line.append(country.name)
++
++ # Format the output
++ line = " ".join(line)
++
++ # Print the output
++ print(line)
++
+ def handle_list_networks_by_as(self, db, ns):
+ writer = self.__get_output_formatter(ns)
+
--- /dev/null
+commit 864dd22e17f7487a90e165274cf3f7898966028d
+Author: Michael Tremer <michael.tremer@ipfire.org>
+Date: Fri Jun 5 10:01:47 2020 +0000
+
+ database: Fix broken search for networks with flags
+
+ The search was ended after the first network. No matter if
+ it matched, or not.
+
+ Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+
+diff --git a/src/database.c b/src/database.c
+index 8e6c5ab..fa1dad0 100644
+--- a/src/database.c
++++ b/src/database.c
+@@ -1208,6 +1208,8 @@ LOC_EXPORT int loc_database_enumerator_next_network(
+ !loc_network_match_flag(*network, enumerator->flags)) {
+ loc_network_unref(*network);
+ *network = NULL;
++
++ continue;
+ }
+
+ return 0;
--- /dev/null
+commit dc1df0f469668ef3dc4e1a9a7623a0ebba2b051e
+Author: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed Jun 3 17:15:27 2020 +0000
+
+ downloader: Change user-agent to location
+
+ Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+
+diff --git a/src/python/downloader.py b/src/python/downloader.py
+index c9e6e00..eb28007 100644
+--- a/src/python/downloader.py
++++ b/src/python/downloader.py
+@@ -71,7 +71,7 @@ class Downloader(object):
+
+ # Update headers
+ headers.update({
+- "User-Agent" : "location-downloader/@VERSION@",
++ "User-Agent" : "location/@VERSION@",
+ })
+
+ # Set headers
--- /dev/null
+commit 10fa313b392a269e15bdaf316218a114d9b23b55
+Author: Michael Tremer <michael.tremer@ipfire.org>
+Date: Fri Jun 5 09:47:36 2020 +0000
+
+ location(8): Export all countries by default
+
+ Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+
+diff --git a/src/python/location.in b/src/python/location.in
+index 5c1effd..6ced5f5 100644
+--- a/src/python/location.in
++++ b/src/python/location.in
+@@ -169,7 +169,7 @@ class CLI(object):
+ export.add_argument("--family",
+ help=_("Specify address family"), choices=("ipv6", "ipv4"),
+ )
+- export.add_argument("objects", nargs="+", help=_("List country codes or ASNs to export"))
++ export.add_argument("objects", nargs="*", help=_("List country codes or ASNs to export"))
+ export.set_defaults(func=self.handle_export)
+
+ args = parser.parse_args()
+@@ -539,9 +539,9 @@ class CLI(object):
+ log.warning("Invalid argument: %s" % object)
+ continue
+
++ # Default to exporting all countries
+ if not countries and not asns:
+- log.error("Nothing to export")
+- return 2
++ countries = ["A1", "A2", "A3"] + [country.code for country in db.countries]
+
+ # Select the output format
+ writer = self.__get_output_formatter(ns)
--- /dev/null
+commit fae36e81a32717ac43c0ce48702f6ff05b7cd029
+Author: Michael Tremer <michael.tremer@ipfire.org>
+Date: Fri Jun 5 09:57:41 2020 +0000
+
+ export flagged networks with their faked country names, too
+
+ This will lead to some networks showing up twice. Once with
+ their real country and once with their faked one.
+
+ It is likely that the first one will match.
+
+ Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+
+diff --git a/src/python/export.py b/src/python/export.py
+index 69fe964..d0bbe77 100644
+--- a/src/python/export.py
++++ b/src/python/export.py
+@@ -23,10 +23,18 @@ import logging
+ import os
+ import socket
+
++import _location
++
+ # Initialise logging
+ log = logging.getLogger("location.export")
+ log.propagate = 1
+
++flags = {
++ _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1",
++ _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2",
++ _location.NETWORK_FLAG_ANYCAST : "A3",
++}
++
+ class OutputWriter(object):
+ suffix = "networks"
+ mode = "w"
+@@ -173,6 +181,17 @@ class Exporter(object):
+ except KeyError:
+ pass
+
++ # Handle flags
++ for flag in flags:
++ if network.has_flag(flag):
++ # Fetch the "fake" country code
++ country = flags[flag]
++
++ try:
++ writers[country].write(network)
++ except KeyError:
++ pass
++
+ # Write everything to the filesystem
+ for writer in writers.values():
+ writer.finish()
--- /dev/null
+commit 141b10999b280b2563580c705d5d23dc4c442deb
+Author: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed Jun 3 16:31:44 2020 +0000
+
+ location-exporter: Do not mistake country AS for an AS number
+
+ Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+
+diff --git a/src/python/location-exporter.in b/src/python/location-exporter.in
+index 894bb44..5454561 100644
+--- a/src/python/location-exporter.in
++++ b/src/python/location-exporter.in
+@@ -22,6 +22,7 @@ import io
+ import ipaddress
+ import logging
+ import os.path
++import re
+ import socket
+ import sys
+
+@@ -258,12 +259,9 @@ class CLI(object):
+ families = [ socket.AF_INET6, socket.AF_INET ]
+
+ for object in ns.objects:
+- if object.startswith("AS"):
+- try:
+- object = int(object[2:])
+- except ValueError:
+- log.error("Invalid argument: %s" % object)
+- return 2
++ m = re.match("^AS(\d+)$", object)
++ if m:
++ object = int(m.group(1))
+
+ asns.append(object)
+
--- /dev/null
+commit 92af07adfb1e06fe1b055fbcf5ba61159637cd73
+Author: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed Jun 3 16:33:44 2020 +0000
+
+ location-exporter: Warn, but do not fail on invalid input
+
+ Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+
+diff --git a/src/python/location-exporter.in b/src/python/location-exporter.in
+index 5454561..d82f1d3 100644
+--- a/src/python/location-exporter.in
++++ b/src/python/location-exporter.in
+@@ -270,8 +270,12 @@ class CLI(object):
+ countries.append(object)
+
+ else:
+- log.error("Invalid argument: %s" % object)
+- return 2
++ log.warning("Invalid argument: %s" % object)
++ continue
++
++ if not countries and not asns:
++ log.error("Nothing to export")
++ return 2
+
+ # Open the database
+ try:
--- /dev/null
+commit 228d0e74ec47c9954d3a0e1da2e1c0fc6c1b518f
+Author: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed Jun 3 16:15:24 2020 +0000
+
+ location-query: Require at least one flag
+
+ Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+
+diff --git a/src/python/location-query.in b/src/python/location-query.in
+index 5f05b5c..dfdff8c 100644
+--- a/src/python/location-query.in
++++ b/src/python/location-query.in
+@@ -246,7 +246,13 @@ class CLI(object):
+ args.family = 0
+
+ # Call function
+- ret = args.func(db, args)
++ try:
++ ret = args.func(db, args)
++
++ # Catch invalid inputs
++ except ValueError as e:
++ sys.stderr.write("%s\n" % e)
++ ret = 2
+
+ # Return with exit code
+ if ret:
+@@ -451,6 +457,9 @@ class CLI(object):
+ if ns.anycast:
+ flags |= location.NETWORK_FLAG_ANYCAST
+
++ if not flags:
++ raise ValueError(_("You must at least pass one flag"))
++
+ with self.__get_output_formatter(ns) as f:
+ for n in db.search_networks(flags=flags, family=ns.family):
+ f.network(n)
--- /dev/null
+commit 889b932aa6172c96872be545af37d351f7c1c705
+Author: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed Jun 3 17:10:35 2020 +0000
+
+ location-downloader: Merge man page into location-query
+
+ Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+
+diff --git a/Makefile.am b/Makefile.am
+index c0b1300..91f0436 100644
+--- a/Makefile.am
++++ b/Makefile.am
+@@ -37,6 +37,9 @@ LIBLOC_CURRENT=0
+ LIBLOC_REVISION=0
+ LIBLOC_AGE=0
+
++pythondir = $(prefix)/lib/python3/dist-packages
++pyexecdir = $(prefix)/lib/python$(PYTHON_VERSION)/lib-dynload
++
+ DISTCHECK_CONFIGURE_FLAGS = \
+ --with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir)
+
+@@ -371,7 +374,6 @@ src_test_signature_LDADD = \
+ # ------------------------------------------------------------------------------
+
+ MANPAGES = \
+- man/location-downloader.8 \
+ man/location-query.8
+
+ MANPAGES_TXT = $(patsubst %.8,%.txt,$(MANPAGES))
+diff --git a/man/location-downloader.txt b/man/location-downloader.txt
+deleted file mode 100644
+index d733923..0000000
+--- a/man/location-downloader.txt
++++ /dev/null
+@@ -1,61 +0,0 @@
+-= location-downloader(8)
+-
+-== NAME
+-location-downloader - Download a location database
+-
+-== SYNOPSIS
+-[verse]
+-`location-downloader update`
+-
+-== DESCRIPTION
+-The `location-downloader` command updates the local version of the
+-location database.
+-
+-== OPTIONS
+-
+---database FILE::
+--d FILE::
+- The path of the database which is being updated.
+- +
+- If this option is omitted, the system's database will be opened.
+-
+---quiet::
+- Enable quiet mode
+-
+---debug::
+- Enable debugging mode
+-
+-== COMMANDS
+-
+-'update'::
+- This command will try to update the local database.
+- +
+- It will terminate with a return code of zero if the database has been
+- successfully updated. 1 on error, 2 on invalid call and 3 if the
+- database was already the latest version.
+-
+-'verify'::
+- Verifies the downloaded database.
+-
+-'--help'::
+- Shows a short help text on using this program.
+-
+-'--version'::
+- Shows the program's version and exists.
+-
+-== EXIT CODES
+-The 'location-downloader' command will normally exit with code zero.
+-If there has been a problem and the requested action could not be performed,
+-the exit code is unequal to zero.
+-
+-== HOW IT WORKS
+-The downloader checks a DNS record for the latest version of the database.
+-It will then try to download a file with that version from a mirror server.
+-If the downloaded file is outdated, the next mirror will be tried until we
+-have found a file that is recent enough.
+-
+-== BUGS
+-Please report all bugs to the bugtracker at https://bugzilla.ipfire.org/.
+-
+-== AUTHORS
+-Michael Tremer
+diff --git a/man/location-query.txt b/man/location-query.txt
+index b91e8e1..acb43cd 100644
+--- a/man/location-query.txt
++++ b/man/location-query.txt
+@@ -8,6 +8,8 @@ location-query - Query the location database
+ `location-query lookup ADDRESS [ADDRESS...]`
+ `location-query get-as ASN [ASN...]`
+ `location-query search-as STRING`
++`location-query update`
++`location-query verify`
+ `location-query list-networks-by-as ASN`
+ `location-query list-networks-by-cc COUNTRY_CODE`
+ `location-query list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]`
+@@ -47,6 +49,16 @@ or countries.
+ +
+ The search will be performed case-insensitively.
+
++'update'::
++ This command will try to update the local database.
++ +
++ It will terminate with a return code of zero if the database has been
++ successfully updated. 1 on error, 2 on invalid call and 3 if the
++ database was already the latest version.
++
++'verify'::
++ Verifies the downloaded database.
++
+ 'list-networks-by-as [--family=[ipv6|ipv4]]Â [--output-format FORMAT]Â ASN'::
+ Lists all networks which belong to this Autonomous System.
+ +
+@@ -85,6 +97,12 @@ The 'location-query' command will normally exit with code zero.
+ If there has been a problem and the requested action could not be performed,
+ the exit code is unequal to zero.
+
++== HOW IT WORKS
++The downloader checks a DNS record for the latest version of the database.
++It will then try to download a file with that version from a mirror server.
++If the downloaded file is outdated, the next mirror will be tried until we
++have found a file that is recent enough.
++
+ == BUGS
+ Please report all bugs to the bugtracker at https://bugzilla.ipfire.org/.
+
--- /dev/null
+commit 88ef7e9cd4b3a1a5662c7dc071bd7a44e1242cba
+Author: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed Jun 3 18:36:28 2020 +0000
+
+ Merge location-exporter(8) into location(8)
+
+ Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+
+diff --git a/Makefile.am b/Makefile.am
+index 59870b1..9f520cc 100644
+--- a/Makefile.am
++++ b/Makefile.am
+@@ -150,6 +150,7 @@ dist_pkgpython_PYTHON = \
+ src/python/__init__.py \
+ src/python/database.py \
+ src/python/downloader.py \
++ src/python/export.py \
+ src/python/i18n.py \
+ src/python/importer.py \
+ src/python/logger.py
+@@ -239,17 +240,14 @@ uninstall-perl:
+
+ bin_SCRIPTS = \
+ src/python/location \
+- src/python/location-exporter \
+ src/python/location-importer
+
+ EXTRA_DIST += \
+ src/python/location.in \
+- src/python/location-exporter.in \
+ src/python/location-importer.in
+
+ CLEANFILES += \
+ src/python/location \
+- src/python/location-exporter \
+ src/python/location-importer
+
+ # ------------------------------------------------------------------------------
+diff --git a/src/python/export.py b/src/python/export.py
+new file mode 100644
+index 0000000..69fe964
+--- /dev/null
++++ b/src/python/export.py
+@@ -0,0 +1,185 @@
++#!/usr/bin/python3
++###############################################################################
++# #
++# libloc - A library to determine the location of someone on the Internet #
++# #
++# Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
++# #
++# This library is free software; you can redistribute it and/or #
++# modify it under the terms of the GNU Lesser General Public #
++# License as published by the Free Software Foundation; either #
++# version 2.1 of the License, or (at your option) any later version. #
++# #
++# This library is distributed in the hope that it will be useful, #
++# but WITHOUT ANY WARRANTY; without even the implied warranty of #
++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
++# Lesser General Public License for more details. #
++# #
++###############################################################################
++
++import io
++import ipaddress
++import logging
++import os
++import socket
++
++# Initialise logging
++log = logging.getLogger("location.export")
++log.propagate = 1
++
++class OutputWriter(object):
++ suffix = "networks"
++ mode = "w"
++
++ def __init__(self, f, prefix=None):
++ self.f, self.prefix = f, prefix
++
++ # Immediately write the header
++ self._write_header()
++
++ @classmethod
++ def open(cls, filename, **kwargs):
++ """
++ Convenience function to open a file
++ """
++ f = open(filename, cls.mode)
++
++ return cls(f, **kwargs)
++
++ def __repr__(self):
++ return "<%s f=%s>" % (self.__class__.__name__, self.f)
++
++ def _write_header(self):
++ """
++ The header of the file
++ """
++ pass
++
++ def _write_footer(self):
++ """
++ The footer of the file
++ """
++ pass
++
++ def write(self, network):
++ self.f.write("%s\n" % network)
++
++ def finish(self):
++ """
++ Called when all data has been written
++ """
++ self._write_footer()
++
++ # Close the file
++ self.f.close()
++
++
++class IpsetOutputWriter(OutputWriter):
++ """
++ For ipset
++ """
++ suffix = "ipset"
++
++ def _write_header(self):
++ self.f.write("create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.prefix)
++
++ def write(self, network):
++ self.f.write("add %s %s\n" % (self.prefix, network))
++
++
++class NftablesOutputWriter(OutputWriter):
++ """
++ For nftables
++ """
++ suffix = "set"
++
++ def _write_header(self):
++ self.f.write("define %s = {\n" % self.prefix)
++
++ def _write_footer(self):
++ self.f.write("}\n")
++
++ def write(self, network):
++ self.f.write(" %s,\n" % network)
++
++
++class XTGeoIPOutputWriter(OutputWriter):
++ """
++ Formats the output in that way, that it can be loaded by
++ the xt_geoip kernel module from xtables-addons.
++ """
++ suffix = "iv"
++ mode = "wb"
++
++ def write(self, network):
++ n = ipaddress.ip_network("%s" % network)
++
++ for address in (n.network_address, n.broadcast_address):
++ bytes = socket.inet_pton(
++ socket.AF_INET6 if address.version == 6 else socket.AF_INET,
++ "%s" % address,
++ )
++
++ self.f.write(bytes)
++
++
++formats = {
++ "ipset" : IpsetOutputWriter,
++ "list" : OutputWriter,
++ "nftables" : NftablesOutputWriter,
++ "xt_geoip" : XTGeoIPOutputWriter,
++}
++
++class Exporter(object):
++ def __init__(self, db, writer):
++ self.db, self.writer = db, writer
++
++ def export(self, directory, families, countries, asns):
++ for family in families:
++ log.debug("Exporting family %s" % family)
++
++ writers = {}
++
++ # Create writers for countries
++ for country_code in countries:
++ filename = self._make_filename(
++ directory, prefix=country_code, suffix=self.writer.suffix, family=family,
++ )
++
++ writers[country_code] = self.writer.open(filename, prefix="CC_%s" % country_code)
++
++ # Create writers for ASNs
++ for asn in asns:
++ filename = self._make_filename(
++ directory, "AS%s" % asn, suffix=self.writer.suffix, family=family,
++ )
++
++ writers[asn] = self.writer.open(filename, prefix="AS%s" % asn)
++
++ # Get all networks that match the family
++ networks = self.db.search_networks(family=family)
++
++ # Walk through all networks
++ for network in networks:
++ # Write matching countries
++ try:
++ writers[network.country_code].write(network)
++ except KeyError:
++ pass
++
++ # Write matching ASNs
++ try:
++ writers[network.asn].write(network)
++ except KeyError:
++ pass
++
++ # Write everything to the filesystem
++ for writer in writers.values():
++ writer.finish()
++
++ def _make_filename(self, directory, prefix, suffix, family):
++ filename = "%s.%s%s" % (
++ prefix, suffix, "6" if family == socket.AF_INET6 else "4"
++ )
++
++ return os.path.join(directory, filename)
+diff --git a/src/python/location-exporter.in b/src/python/location-exporter.in
+deleted file mode 100644
+index d82f1d3..0000000
+--- a/src/python/location-exporter.in
++++ /dev/null
+@@ -1,300 +0,0 @@
+-#!/usr/bin/python3
+-###############################################################################
+-# #
+-# libloc - A library to determine the location of someone on the Internet #
+-# #
+-# Copyright (C) 2019 IPFire Development Team <info@ipfire.org> #
+-# #
+-# This library is free software; you can redistribute it and/or #
+-# modify it under the terms of the GNU Lesser General Public #
+-# License as published by the Free Software Foundation; either #
+-# version 2.1 of the License, or (at your option) any later version. #
+-# #
+-# This library is distributed in the hope that it will be useful, #
+-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
+-# Lesser General Public License for more details. #
+-# #
+-###############################################################################
+-
+-import argparse
+-import io
+-import ipaddress
+-import logging
+-import os.path
+-import re
+-import socket
+-import sys
+-
+-# Load our location module
+-import location
+-from location.i18n import _
+-
+-# Initialise logging
+-log = logging.getLogger("location.exporter")
+-log.propagate = 1
+-
+-class OutputWriter(object):
+- suffix = "networks"
+-
+- def __init__(self, family, country_code=None, asn=None):
+- self.family, self.country_code, self.asn = family, country_code, asn
+-
+- self.f = io.BytesIO()
+-
+- def write_out(self, directory):
+- # Make the output filename
+- filename = os.path.join(
+- directory, self._make_filename(),
+- )
+-
+- with open(filename, "wb") as f:
+- self._write_header(f)
+-
+- # Copy all data into the file
+- f.write(self.f.getbuffer())
+-
+- self._write_footer(f)
+-
+- def _make_filename(self):
+- return "%s.%s%s" % (
+- self.country_code or "AS%s" % self.asn,
+- self.suffix,
+- "6" if self.family == socket.AF_INET6 else "4"
+- )
+-
+- @property
+- def name(self):
+- if self.country_code:
+- return "CC_%s" % self.country_code
+-
+- if self.asn:
+- return "AS%s" % self.asn
+-
+- def _write_header(self, f):
+- """
+- The header of the file
+- """
+- pass
+-
+- def _write_footer(self, f):
+- """
+- The footer of the file
+- """
+- pass
+-
+- def write(self, network):
+- s = "%s\n" % network
+-
+- self.f.write(s.encode("ascii"))
+-
+-
+-class IpsetOutputWriter(OutputWriter):
+- """
+- For ipset
+- """
+- suffix = "ipset"
+-
+- def _write_header(self, f):
+- h = "create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.name
+-
+- f.write(h.encode("ascii"))
+-
+- def write(self, network):
+- s = "add %s %s\n" % (self.name, network)
+-
+- self.f.write(s.encode("ascii"))
+-
+-
+-class NftablesOutputWriter(OutputWriter):
+- """
+- For nftables
+- """
+- suffix = "set"
+-
+- def _write_header(self, f):
+- h = "define %s = {\n" % self.name
+-
+- f.write(h.encode("ascii"))
+-
+- def _write_footer(self, f):
+- f.write(b"}")
+-
+- def write(self, network):
+- s = " %s,\n" % network
+-
+- self.f.write(s.encode("ascii"))
+-
+-
+-class XTGeoIPOutputWriter(OutputWriter):
+- """
+- Formats the output in that way, that it can be loaded by
+- the xt_geoip kernel module from xtables-addons.
+- """
+- suffix = "iv"
+-
+- def write(self, network):
+- n = ipaddress.ip_network("%s" % network)
+-
+- for address in (n.network_address, n.broadcast_address):
+- bytes = socket.inet_pton(
+- socket.AF_INET6 if address.version == 6 else socket.AF_INET,
+- "%s" % address,
+- )
+-
+- self.f.write(bytes)
+-
+-
+-class Exporter(object):
+- def __init__(self, db, writer):
+- self.db = db
+- self.writer = writer
+-
+- def export(self, directory, families, countries, asns):
+- for family in families:
+- log.debug("Exporting family %s" % family)
+-
+- writers = {}
+-
+- # Create writers for countries
+- for country_code in countries:
+- writers[country_code] = self.writer(family, country_code=country_code)
+-
+- # Create writers for ASNs
+- for asn in asns:
+- writers[asn] = self.writer(family, asn=asn)
+-
+- # Get all networks that match the family
+- networks = self.db.search_networks(family=family)
+-
+- # Walk through all networks
+- for network in networks:
+- # Write matching countries
+- if network.country_code in countries:
+- writers[network.country_code].write(network)
+-
+- # Write matching ASNs
+- if network.asn in asns:
+- writers[network.asn].write(network)
+-
+- # Write everything to the filesystem
+- for writer in writers.values():
+- writer.write_out(directory)
+-
+-
+-class CLI(object):
+- output_formats = {
+- "ipset" : IpsetOutputWriter,
+- "list" : OutputWriter,
+- "nftables" : NftablesOutputWriter,
+- "xt_geoip" : XTGeoIPOutputWriter,
+- }
+-
+- def parse_cli(self):
+- parser = argparse.ArgumentParser(
+- description=_("Location Exporter Command Line Interface"),
+- )
+-
+- # Global configuration flags
+- parser.add_argument("--debug", action="store_true",
+- help=_("Enable debug output"))
+- parser.add_argument("--quiet", action="store_true",
+- help=_("Enable quiet mode"))
+-
+- # version
+- parser.add_argument("--version", action="version",
+- version="%(prog)s @VERSION@")
+-
+- # database
+- parser.add_argument("--database", "-d",
+- default="@databasedir@/database.db", help=_("Path to database"),
+- )
+-
+- # format
+- parser.add_argument("--format", help=_("Output format"),
+- default="list", choices=self.output_formats.keys())
+-
+- # directory
+- parser.add_argument("--directory", help=_("Output directory"), required=True)
+-
+- # family
+- parser.add_argument("--family", help=_("Specify address family"), choices=("ipv6", "ipv4"))
+-
+- # Countries and Autonomous Systems
+- parser.add_argument("objects", nargs="+")
+-
+- args = parser.parse_args()
+-
+- # Configure logging
+- if args.debug:
+- location.logger.set_level(logging.DEBUG)
+- elif args.quiet:
+- location.logger.set_level(logging.WARNING)
+-
+- return args
+-
+- def run(self):
+- # Parse command line arguments
+- args = self.parse_cli()
+-
+- # Call function
+- ret = self.handle_export(args)
+-
+- # Return with exit code
+- if ret:
+- sys.exit(ret)
+-
+- # Otherwise just exit
+- sys.exit(0)
+-
+- def handle_export(self, ns):
+- countries, asns = [], []
+-
+- # Translate family
+- if ns.family == "ipv6":
+- families = [ socket.AF_INET6 ]
+- elif ns.family == "ipv4":
+- families = [ socket.AF_INET ]
+- else:
+- families = [ socket.AF_INET6, socket.AF_INET ]
+-
+- for object in ns.objects:
+- m = re.match("^AS(\d+)$", object)
+- if m:
+- object = int(m.group(1))
+-
+- asns.append(object)
+-
+- elif location.country_code_is_valid(object) \
+- or object in ("A1", "A2", "A3"):
+- countries.append(object)
+-
+- else:
+- log.warning("Invalid argument: %s" % object)
+- continue
+-
+- if not countries and not asns:
+- log.error("Nothing to export")
+- return 2
+-
+- # Open the database
+- try:
+- db = location.Database(ns.database)
+- except FileNotFoundError as e:
+- log.error("Count not open database: %s" % ns.database)
+- return 1
+-
+- # Select the output format
+- writer = self.output_formats.get(ns.format)
+- assert writer
+-
+- e = Exporter(db, writer)
+- e.export(ns.directory, countries=countries, asns=asns, families=families)
+-
+-
+-def main():
+- # Run the command line interface
+- c = CLI()
+- c.run()
+-
+-main()
+diff --git a/src/python/location.in b/src/python/location.in
+index 10618e2..7614cae 100644
+--- a/src/python/location.in
++++ b/src/python/location.in
+@@ -22,6 +22,7 @@ import datetime
+ import ipaddress
+ import logging
+ import os
++import re
+ import shutil
+ import socket
+ import sys
+@@ -30,6 +31,8 @@ import time
+ # Load our location module
+ import location
+ import location.downloader
++import location.export
++
+ from location.i18n import _
+
+ # Setup logging
+@@ -37,88 +40,7 @@ log = logging.getLogger("location")
+
+ # Output formatters
+
+-class OutputFormatter(object):
+- def __init__(self, ns):
+- self.ns = ns
+-
+- def __enter__(self):
+- # Open the output
+- self.open()
+-
+- return self
+-
+- def __exit__(self, type, value, tb):
+- if tb is None:
+- self.close()
+-
+- @property
+- def name(self):
+- if "country_code" in self.ns:
+- return "networks_country_%s" % self.ns.country_code[0]
+-
+- elif "asn" in self.ns:
+- return "networks_AS%s" % self.ns.asn[0]
+-
+- def open(self):
+- pass
+-
+- def close(self):
+- pass
+-
+- def network(self, network):
+- print(network)
+-
+-
+-class IpsetOutputFormatter(OutputFormatter):
+- """
+- For nftables
+- """
+- def open(self):
+- print("create %s hash:net family inet hashsize 1024 maxelem 65536" % self.name)
+-
+- def network(self, network):
+- print("add %s %s" % (self.name, network))
+-
+-
+-class NftablesOutputFormatter(OutputFormatter):
+- """
+- For nftables
+- """
+- def open(self):
+- print("define %s = {" % self.name)
+-
+- def close(self):
+- print("}")
+-
+- def network(self, network):
+- print(" %s," % network)
+-
+-
+-class XTGeoIPOutputFormatter(OutputFormatter):
+- """
+- Formats the output in that way, that it can be loaded by
+- the xt_geoip kernel module from xtables-addons.
+- """
+- def network(self, network):
+- n = ipaddress.ip_network("%s" % network)
+-
+- for address in (n.network_address, n.broadcast_address):
+- bytes = socket.inet_pton(
+- socket.AF_INET6 if address.version == 6 else socket.AF_INET,
+- "%s" % address,
+- )
+-
+- os.write(1, bytes)
+-
+-
+ class CLI(object):
+- output_formats = {
+- "ipset" : IpsetOutputFormatter,
+- "list" : OutputFormatter,
+- "nftables" : NftablesOutputFormatter,
+- "xt_geoip" : XTGeoIPOutputFormatter,
+- }
+-
+ def parse_cli(self):
+ parser = argparse.ArgumentParser(
+ description=_("Location Database Command Line Interface"),
+@@ -193,8 +115,8 @@ class CLI(object):
+ )
+ list_networks_by_as.add_argument("asn", nargs=1, type=int)
+ list_networks_by_as.add_argument("--family", choices=("ipv6", "ipv4"))
+- list_networks_by_as.add_argument("--output-format",
+- choices=self.output_formats.keys(), default="list")
++ list_networks_by_as.add_argument("--format",
++ choices=location.export.formats.keys(), default="list")
+ list_networks_by_as.set_defaults(func=self.handle_list_networks_by_as)
+
+ # List all networks in a country
+@@ -203,8 +125,8 @@ class CLI(object):
+ )
+ list_networks_by_cc.add_argument("country_code", nargs=1)
+ list_networks_by_cc.add_argument("--family", choices=("ipv6", "ipv4"))
+- list_networks_by_cc.add_argument("--output-format",
+- choices=self.output_formats.keys(), default="list")
++ list_networks_by_cc.add_argument("--format",
++ choices=location.export.formats.keys(), default="list")
+ list_networks_by_cc.set_defaults(func=self.handle_list_networks_by_cc)
+
+ # List all networks with flags
+@@ -221,10 +143,23 @@ class CLI(object):
+ action="store_true", help=_("Anycasts"),
+ )
+ list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4"))
+- list_networks_by_flags.add_argument("--output-format",
+- choices=self.output_formats.keys(), default="list")
++ list_networks_by_flags.add_argument("--format",
++ choices=location.export.formats.keys(), default="list")
+ list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags)
+
++ # Export
++ export = subparsers.add_parser("export",
++ help=_("Exports data in many formats to load it into packet filters"),
++ )
++ export.add_argument("--format", help=_("Output format"),
++ choices=location.export.formats.keys(), default="list")
++ export.add_argument("--directory", help=_("Output directory"), required=True)
++ export.add_argument("--family",
++ help=_("Specify address family"), choices=("ipv6", "ipv4"),
++ )
++ export.add_argument("objects", nargs="+", help=_("List country codes or ASNs to export"))
++ export.set_defaults(func=self.handle_export)
++
+ args = parser.parse_args()
+
+ # Configure logging
+@@ -494,25 +429,36 @@ class CLI(object):
+
+ def __get_output_formatter(self, ns):
+ try:
+- cls = self.output_formats[ns.output_format]
++ cls = location.export.formats[ns.format]
+ except KeyError:
+- cls = OutputFormatter
++ cls = location.export.OutputFormatter
+
+- return cls(ns)
++ return cls
+
+ def handle_list_networks_by_as(self, db, ns):
+- with self.__get_output_formatter(ns) as f:
+- for asn in ns.asn:
+- # Print all matching networks
+- for n in db.search_networks(asn=asn, family=ns.family):
+- f.network(n)
++ writer = self.__get_output_formatter(ns)
++
++ for asn in ns.asn:
++ f = writer(sys.stdout, prefix="AS%s" % asn)
++
++ # Print all matching networks
++ for n in db.search_networks(asn=asn, family=ns.family):
++ f.write(n)
++
++ f.finish()
+
+ def handle_list_networks_by_cc(self, db, ns):
+- with self.__get_output_formatter(ns) as f:
+- for country_code in ns.country_code:
+- # Print all matching networks
+- for n in db.search_networks(country_code=country_code, family=ns.family):
+- f.network(n)
++ writer = self.__get_output_formatter(ns)
++
++ for country_code in ns.country_code:
++ # Open standard output
++ f = writer(sys.stdout, prefix=country_code)
++
++ # Print all matching networks
++ for n in db.search_networks(country_code=country_code, family=ns.family):
++ f.write(n)
++
++ f.finish()
+
+ def handle_list_networks_by_flags(self, db, ns):
+ flags = 0
+@@ -529,9 +475,49 @@ class CLI(object):
+ if not flags:
+ raise ValueError(_("You must at least pass one flag"))
+
+- with self.__get_output_formatter(ns) as f:
+- for n in db.search_networks(flags=flags, family=ns.family):
+- f.network(n)
++ writer = self.__get_output_formatter(ns)
++ f = writer(sys.stdout, prefix="custom")
++
++ for n in db.search_networks(flags=flags, family=ns.family):
++ f.write(n)
++
++ f.finish()
++
++ def handle_export(self, db, ns):
++ countries, asns = [], []
++
++ # Translate family
++ if ns.family == "ipv6":
++ families = [ socket.AF_INET6 ]
++ elif ns.family == "ipv4":
++ families = [ socket.AF_INET ]
++ else:
++ families = [ socket.AF_INET6, socket.AF_INET ]
++
++ for object in ns.objects:
++ m = re.match("^AS(\d+)$", object)
++ if m:
++ object = int(m.group(1))
++
++ asns.append(object)
++
++ elif location.country_code_is_valid(object) \
++ or object in ("A1", "A2", "A3"):
++ countries.append(object)
++
++ else:
++ log.warning("Invalid argument: %s" % object)
++ continue
++
++ if not countries and not asns:
++ log.error("Nothing to export")
++ return 2
++
++ # Select the output format
++ writer = self.__get_output_formatter(ns)
++
++ e = location.export.Exporter(db, writer)
++ e.export(ns.directory, countries=countries, asns=asns, families=families)
+
+
+ def main():
--- /dev/null
+commit a6f1e3463d4c2085c203ad58072d7a154b663904
+Author: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed Jun 3 17:06:13 2020 +0000
+
+ Move location-downloader functionality into location-query
+
+ The commands are very long and confusion. Hence we merge this
+ all into one command.
+
+ Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+
+diff --git a/Makefile.am b/Makefile.am
+index 31869e0..c0b1300 100644
+--- a/Makefile.am
++++ b/Makefile.am
+@@ -146,6 +146,7 @@ CLEANFILES += \
+ dist_pkgpython_PYTHON = \
+ src/python/__init__.py \
+ src/python/database.py \
++ src/python/downloader.py \
+ src/python/i18n.py \
+ src/python/importer.py \
+ src/python/logger.py
+@@ -234,19 +235,16 @@ uninstall-perl:
+ $(DESTDIR)/$(prefix)/man/man3/Location.3pm
+
+ bin_SCRIPTS = \
+- src/python/location-downloader \
+ src/python/location-exporter \
+ src/python/location-importer \
+ src/python/location-query
+
+ EXTRA_DIST += \
+- src/python/location-downloader.in \
+ src/python/location-exporter.in \
+ src/python/location-importer.in \
+ src/python/location-query.in
+
+ CLEANFILES += \
+- src/python/location-downloader \
+ src/python/location-exporter \
+ src/python/location-importer \
+ src/python/location-query
+diff --git a/src/python/location-downloader.in b/src/python/downloader.py
+similarity index 60%
+rename from src/python/location-downloader.in
+rename to src/python/downloader.py
+index bf0d682..c9e6e00 100644
+--- a/src/python/location-downloader.in
++++ b/src/python/downloader.py
+@@ -3,7 +3,7 @@
+ # #
+ # libloc - A library to determine the location of someone on the Internet #
+ # #
+-# Copyright (C) 2019 IPFire Development Team <info@ipfire.org> #
++# Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
+ # #
+ # This library is free software; you can redistribute it and/or #
+ # modify it under the terms of the GNU Lesser General Public #
+@@ -17,24 +17,18 @@
+ # #
+ ###############################################################################
+
+-import argparse
+-import datetime
+ import logging
+ import lzma
+ import os
+ import random
+-import shutil
+ import stat
+-import sys
+ import tempfile
+ import time
+ import urllib.error
+ import urllib.parse
+ import urllib.request
+
+-# Load our location module
+-import location
+-from location.i18n import _
++from _location import Database, DATABASE_VERSION_LATEST
+
+ DATABASE_FILENAME = "location.db.xz"
+ MIRRORS = (
+@@ -46,9 +40,11 @@ log = logging.getLogger("location.downloader")
+ log.propagate = 1
+
+ class Downloader(object):
+- def __init__(self, version, mirrors):
++ def __init__(self, version=DATABASE_VERSION_LATEST, mirrors=None):
+ self.version = version
+- self.mirrors = list(mirrors)
++
++ # Set mirrors or use defaults
++ self.mirrors = list(mirrors or MIRRORS)
+
+ # Randomize mirrors
+ random.shuffle(self.mirrors)
+@@ -117,9 +113,10 @@ class Downloader(object):
+
+ return res
+
+- def download(self, url, public_key, timestamp=None, tmpdir=None, **kwargs):
+- headers = {}
++ def download(self, public_key, timestamp=None, tmpdir=None, **kwargs):
++ url = "%s/%s" % (self.version, DATABASE_FILENAME)
+
++ headers = {}
+ if timestamp:
+ headers["If-Modified-Since"] = timestamp.strftime(
+ "%a, %d %b %Y %H:%M:%S GMT",
+@@ -191,7 +188,7 @@ class Downloader(object):
+ """
+ log.debug("Opening downloaded database at %s" % f.name)
+
+- db = location.Database(f.name)
++ db = Database(f.name)
+
+ # Database is not recent
+ if timestamp and db.created_at < timestamp.timestamp():
+@@ -208,141 +205,3 @@ class Downloader(object):
+ return False
+
+ return True
+-
+-
+-class CLI(object):
+- def __init__(self):
+- # Which version are we downloading?
+- self.version = location.DATABASE_VERSION_LATEST
+-
+- self.downloader = Downloader(version=self.version, mirrors=MIRRORS)
+-
+- def parse_cli(self):
+- parser = argparse.ArgumentParser(
+- description=_("Location Downloader Command Line Interface"),
+- )
+- subparsers = parser.add_subparsers()
+-
+- # Global configuration flags
+- parser.add_argument("--debug", action="store_true",
+- help=_("Enable debug output"))
+- parser.add_argument("--quiet", action="store_true",
+- help=_("Enable quiet mode"))
+-
+- # version
+- parser.add_argument("--version", action="version",
+- version="%(prog)s @VERSION@")
+-
+- # database
+- parser.add_argument("--database", "-d",
+- default="@databasedir@/database.db", help=_("Path to database"),
+- )
+-
+- # public key
+- parser.add_argument("--public-key", "-k",
+- default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
+- )
+-
+- # Update
+- update = subparsers.add_parser("update", help=_("Update database"))
+- update.set_defaults(func=self.handle_update)
+-
+- # Verify
+- verify = subparsers.add_parser("verify",
+- help=_("Verify the downloaded database"))
+- verify.set_defaults(func=self.handle_verify)
+-
+- args = parser.parse_args()
+-
+- # Configure logging
+- if args.debug:
+- location.logger.set_level(logging.DEBUG)
+- elif args.quiet:
+- location.logger.set_level(logging.WARNING)
+-
+- # Print usage if no action was given
+- if not "func" in args:
+- parser.print_usage()
+- sys.exit(2)
+-
+- return args
+-
+- def run(self):
+- # Parse command line arguments
+- args = self.parse_cli()
+-
+- # Call function
+- ret = args.func(args)
+-
+- # Return with exit code
+- if ret:
+- sys.exit(ret)
+-
+- # Otherwise just exit
+- sys.exit(0)
+-
+- def handle_update(self, ns):
+- # Fetch the timestamp we need from DNS
+- t = location.discover_latest_version(self.version)
+-
+- # Parse timestamp into datetime format
+- timestamp = datetime.datetime.fromtimestamp(t) if t else None
+-
+- # Open database
+- try:
+- db = location.Database(ns.database)
+-
+- # Check if we are already on the latest version
+- if timestamp and db.created_at >= timestamp.timestamp():
+- log.info("Already on the latest version")
+- return
+-
+- except FileNotFoundError as e:
+- db = None
+-
+- # Download the database into the correct directory
+- tmpdir = os.path.dirname(ns.database)
+-
+- # Try downloading a new database
+- try:
+- t = self.downloader.download("%s/%s" % (self.version, DATABASE_FILENAME),
+- public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
+-
+- # If no file could be downloaded, log a message
+- except FileNotFoundError as e:
+- log.error("Could not download a new database")
+- return 1
+-
+- # If we have not received a new file, there is nothing to do
+- if not t:
+- return 3
+-
+- # Move temporary file to destination
+- shutil.move(t.name, ns.database)
+-
+- return 0
+-
+- def handle_verify(self, ns):
+- try:
+- db = location.Database(ns.database)
+- except FileNotFoundError as e:
+- log.error("%s: %s" % (ns.database, e))
+- return 127
+-
+- # Verify the database
+- with open(ns.public_key, "r") as f:
+- if not db.verify(f):
+- log.error("Could not verify database")
+- return 1
+-
+- # Success
+- log.debug("Database successfully verified")
+- return 0
+-
+-
+-def main():
+- # Run the command line interface
+- c = CLI()
+- c.run()
+-
+-main()
+diff --git a/src/python/location-query.in b/src/python/location-query.in
+index dfdff8c..0291786 100644
+--- a/src/python/location-query.in
++++ b/src/python/location-query.in
+@@ -18,16 +18,23 @@
+ ###############################################################################
+
+ import argparse
++import datetime
+ import ipaddress
++import logging
+ import os
++import shutil
+ import socket
+ import sys
+ import time
+
+ # Load our location module
+ import location
++import location.downloader
+ from location.i18n import _
+
++# Setup logging
++log = logging.getLogger("location")
++
+ # Output formatters
+
+ class OutputFormatter(object):
+@@ -157,6 +164,15 @@ class CLI(object):
+ dump.add_argument("output", nargs="?", type=argparse.FileType("w"))
+ dump.set_defaults(func=self.handle_dump)
+
++ # Update
++ update = subparsers.add_parser("update", help=_("Update database"))
++ update.set_defaults(func=self.handle_update)
++
++ # Verify
++ verify = subparsers.add_parser("verify",
++ help=_("Verify the downloaded database"))
++ verify.set_defaults(func=self.handle_verify)
++
+ # Get AS
+ get_as = subparsers.add_parser("get-as",
+ help=_("Get information about one or multiple Autonomous Systems"),
+@@ -423,6 +439,59 @@ class CLI(object):
+ for a in db.search_as(query):
+ print(a)
+
++ def handle_update(self, db, ns):
++ # Fetch the timestamp we need from DNS
++ t = location.discover_latest_version()
++
++ # Parse timestamp into datetime format
++ timestamp = datetime.datetime.fromtimestamp(t) if t else None
++
++ # Check the version of the local database
++ if db and timestamp and db.created_at >= timestamp.timestamp():
++ log.info("Already on the latest version")
++ return
++
++ # Download the database into the correct directory
++ tmpdir = os.path.dirname(ns.database)
++
++ # Create a downloader
++ d = location.downloader.Downloader()
++
++ # Try downloading a new database
++ try:
++ t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
++
++ # If no file could be downloaded, log a message
++ except FileNotFoundError as e:
++ log.error("Could not download a new database")
++ return 1
++
++ # If we have not received a new file, there is nothing to do
++ if not t:
++ return 3
++
++ # Move temporary file to destination
++ shutil.move(t.name, ns.database)
++
++ return 0
++
++ def handle_verify(self, ns):
++ try:
++ db = location.Database(ns.database)
++ except FileNotFoundError as e:
++ log.error("%s: %s" % (ns.database, e))
++ return 127
++
++ # Verify the database
++ with open(ns.public_key, "r") as f:
++ if not db.verify(f):
++ log.error("Could not verify database")
++ return 1
++
++ # Success
++ log.debug("Database successfully verified")
++ return 0
++
+ def __get_output_formatter(self, ns):
+ try:
+ cls = self.output_formats[ns.output_format]
+diff --git a/src/python/locationmodule.c b/src/python/locationmodule.c
+index a04cab7..5b72be9 100644
+--- a/src/python/locationmodule.c
++++ b/src/python/locationmodule.c
+@@ -50,9 +50,9 @@ static PyObject* set_log_level(PyObject* m, PyObject* args) {
+ }
+
+ static PyObject* discover_latest_version(PyObject* m, PyObject* args) {
+- unsigned int version = 0;
++ unsigned int version = LOC_DATABASE_VERSION_LATEST;
+
+- if (!PyArg_ParseTuple(args, "i", &version))
++ if (!PyArg_ParseTuple(args, "|i", &version))
+ return NULL;
+
+ time_t t = 0;
--- /dev/null
+commit 6bfde1447d237d2a345b99677c5b74e54cbd5739
+Author: Michael Tremer <michael.tremer@ipfire.org>
+Date: Thu Jun 4 10:37:50 2020 +0000
+
+ Makefile: Remove accidentially committed hacks for Debian
+
+ Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+
+diff --git a/Makefile.am b/Makefile.am
+index ef57551..c75839c 100644
+--- a/Makefile.am
++++ b/Makefile.am
+@@ -37,9 +37,6 @@ LIBLOC_CURRENT=0
+ LIBLOC_REVISION=0
+ LIBLOC_AGE=0
+
+-pythondir = $(prefix)/lib/python3/dist-packages
+-pyexecdir = $(prefix)/lib/python$(PYTHON_VERSION)/lib-dynload
+-
+ DISTCHECK_CONFIGURE_FLAGS = \
+ --with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir)
+
--- /dev/null
+commit 1d237439676e8b9ee10a6dde2c64f5ba3a057210
+Author: Michael Tremer <michael.tremer@ipfire.org>
+Date: Wed Jun 3 17:21:31 2020 +0000
+
+ Rename location-query(8) to location(8)
+
+ Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
+
+diff --git a/Makefile.am b/Makefile.am
+index bf204d8..59870b1 100644
+--- a/Makefile.am
++++ b/Makefile.am
+@@ -238,19 +238,19 @@ uninstall-perl:
+ $(DESTDIR)/$(prefix)/man/man3/Location.3pm
+
+ bin_SCRIPTS = \
++ src/python/location \
+ src/python/location-exporter \
+- src/python/location-importer \
+- src/python/location-query
++ src/python/location-importer
+
+ EXTRA_DIST += \
++ src/python/location.in \
+ src/python/location-exporter.in \
+- src/python/location-importer.in \
+- src/python/location-query.in
++ src/python/location-importer.in
+
+ CLEANFILES += \
++ src/python/location \
+ src/python/location-exporter \
+- src/python/location-importer \
+- src/python/location-query
++ src/python/location-importer
+
+ # ------------------------------------------------------------------------------
+
+@@ -374,7 +374,7 @@ src_test_signature_LDADD = \
+ # ------------------------------------------------------------------------------
+
+ MANPAGES = \
+- man/location-query.8
++ man/location.8
+
+ MANPAGES_TXT = $(patsubst %.8,%.txt,$(MANPAGES))
+ MANPAGES_HTML = $(patsubst %.txt,%.html,$(MANPAGES_TXT))
+diff --git a/man/location-query.txt b/man/location.txt
+similarity index 84%
+rename from man/location-query.txt
+rename to man/location.txt
+index acb43cd..672c2b2 100644
+--- a/man/location-query.txt
++++ b/man/location.txt
+@@ -1,21 +1,21 @@
+-= location-query(8)
++= location(8)
+
+ == NAME
+-location-query - Query the location database
++location - Query the location database
+
+ == SYNOPSIS
+ [verse]
+-`location-query lookup ADDRESS [ADDRESS...]`
+-`location-query get-as ASN [ASN...]`
+-`location-query search-as STRING`
+-`location-query update`
+-`location-query verify`
+-`location-query list-networks-by-as ASN`
+-`location-query list-networks-by-cc COUNTRY_CODE`
+-`location-query list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]`
++`location lookup ADDRESS [ADDRESS...]`
++`location get-as ASN [ASN...]`
++`location search-as STRING`
++`location update`
++`location verify`
++`location list-networks-by-as ASN`
++`location list-networks-by-cc COUNTRY_CODE`
++`location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]`
+
+ == DESCRIPTION
+-The `location-query` retrieves information from the location database.
++`location` retrieves information from the location database.
+ This data can be used to determine someone's location on the Internet
+ and for building firewall rulesets to block access from certain ASes
+ or countries.
+@@ -93,7 +93,7 @@ or countries.
+ Shows the program's version and exists.
+
+ == EXIT CODES
+-The 'location-query' command will normally exit with code zero.
++The 'location' command will normally exit with code zero.
+ If there has been a problem and the requested action could not be performed,
+ the exit code is unequal to zero.
+
+diff --git a/src/python/location-query.in b/src/python/location.in
+similarity index 99%
+rename from src/python/location-query.in
+rename to src/python/location.in
+index 0291786..10618e2 100644
+--- a/src/python/location-query.in
++++ b/src/python/location.in
+@@ -248,7 +248,7 @@ class CLI(object):
+ try:
+ db = location.Database(args.database)
+ except FileNotFoundError as e:
+- sys.stderr.write("location-query: Could not open database %s: %s\n" \
++ sys.stderr.write("location: Could not open database %s: %s\n" \
+ % (args.database, e))
+ sys.exit(1)
projects / ipfire-2.x.git / commitdiff
