include Config
-VER = 0.9.1
+VER = 0.9.2
DB_DATE = 2020-06-10
THISAPP = libloc-$(VER)
objects = $(DL_FILE) \
location-$(DB_DATE).db.xz
-$(DL_FILE) = $(DL_FROM)/$(DL_FILE)
+$(DL_FILE) = https://source.ipfire.org/releases/libloc//$(DL_FILE)
location-$(DB_DATE).db.xz = https://location.ipfire.org/databases/1/archive/location-$(DB_DATE).db.xz
-$(DL_FILE)_MD5 = b62331e7a5bc5299bdd35f340342fc51
+$(DL_FILE)_MD5 = c6ed4fcbdb2ce4ca7e6df8abbd985411
location-$(DB_DATE).db.xz_MD5 = 268b6d58a26c6d36081ed1e899b89020
install : $(TARGET)
@$(PREBUILD)
@rm -rf $(DIR_APP) && cd $(DIR_SRC) && tar xvf $(DIR_DL)/$(DL_FILE)
- # Add upstream patches.
- cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-location-downloader-do-not-change-content-of-open-database.patch
- cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-remove-python-path-overrides-for-debian.patch
- cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-location-query-require-at-least-one-flag.patch
- cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-location-exporter-do-not-mistake-country-as-for-an-as-number.patch
- cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-location-exporter-warn-but-do-not-fail-on-invalid-input.patch
- cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-move-location-downloader-functionality-into-location-query.patch
- cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-merge-location-downloader-manpage-into-location-query.patch
- cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-downloader-rename-user-agent-to-location.patch
- cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-rename-location-query-to-location.patch
- cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-merge-location-exporter-into-location.patch
- cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-remove-accidently-commited-hacks-for-debian.patch
- cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-add-option-to-iterate-over-all-contries-and-print-them.patch
- cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-export-all-countries-by-default.patch
- cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-export-flagged-networks-with-their-faked-country-names-too.patch
- cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-database-fix-brocken-search-for-networks-with-flags.patch
- cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-adjust-format-to-print-ASes.patch
-
cd $(DIR_APP) && ./autogen.sh
cd $(DIR_APP) && ./configure \
--prefix=/usr \
+++ /dev/null
-commit fa9a3663cb2dfb2490da43f6967f1a3a2948fc8a
-Author: Michael Tremer <michael.tremer@ipfire.org>
-Date: Fri Jun 5 09:41:28 2020 +0000
-
- Add option to iterate over all countries and print them to the console
-
- Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
-
-diff --git a/man/location.txt b/man/location.txt
-index 672c2b2..0d70e0b 100644
---- a/man/location.txt
-+++ b/man/location.txt
-@@ -13,6 +13,7 @@ location - Query the location database
- `location list-networks-by-as ASN`
- `location list-networks-by-cc COUNTRY_CODE`
- `location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]`
-+`location list-countries [--show-name] [--show-continent]`
-
- == DESCRIPTION
- `location` retrieves information from the location database.
-@@ -86,6 +87,12 @@ or countries.
- +
- See above for usage of the '--family' and '--output-format' parameters.
-
-+'list-countries [--show-name] [--show-continent]'::
-+ Lists all countries known to the database.
-+ +
-+ With the optional parameters '--show-name' and '--show-continent', the name and
-+ continent code will be printed, too.
-+
- '--help'::
- Shows a short help text on using this program.
-
-diff --git a/src/database.c b/src/database.c
-index d919278..8e6c5ab 100644
---- a/src/database.c
-+++ b/src/database.c
-@@ -107,6 +107,9 @@ struct loc_database_enumerator {
- // Index of the AS we are looking at
- unsigned int as_index;
-
-+ // Index of the country we are looking at
-+ unsigned int country_index;
-+
- // Network state
- struct in6_addr network_address;
- struct loc_node_stack network_stack[MAX_STACK_DEPTH];
-@@ -1219,3 +1222,30 @@ LOC_EXPORT int loc_database_enumerator_next_network(
-
- return 0;
- }
-+
-+LOC_EXPORT int loc_database_enumerator_next_country(
-+ struct loc_database_enumerator* enumerator, struct loc_country** country) {
-+ *country = NULL;
-+
-+ // Do not do anything if not in country mode
-+ if (enumerator->mode != LOC_DB_ENUMERATE_COUNTRIES)
-+ return 0;
-+
-+ struct loc_database* db = enumerator->db;
-+
-+ while (enumerator->country_index < db->countries_count) {
-+ // Fetch the next country
-+ int r = loc_database_fetch_country(db, country, enumerator->country_index++);
-+ if (r)
-+ return r;
-+
-+ // We do not filter here, so it always is a match
-+ return 0;
-+ }
-+
-+ // Reset the index
-+ enumerator->country_index = 0;
-+
-+ // We have searched through all of them
-+ return 0;
-+}
-diff --git a/src/libloc.sym b/src/libloc.sym
-index e9e8549..9a1e6f0 100644
---- a/src/libloc.sym
-+++ b/src/libloc.sym
-@@ -68,6 +68,7 @@ global:
- # Database Enumerator
- loc_database_enumerator_new;
- loc_database_enumerator_next_as;
-+ loc_database_enumerator_next_country;
- loc_database_enumerator_next_network;
- loc_database_enumerator_ref;
- loc_database_enumerator_set_asn;
-diff --git a/src/loc/database.h b/src/loc/database.h
-index ab9ef72..43173dd 100644
---- a/src/loc/database.h
-+++ b/src/loc/database.h
-@@ -50,8 +50,9 @@ int loc_database_get_country(struct loc_database* db,
- struct loc_country** country, const char* code);
-
- enum loc_database_enumerator_mode {
-- LOC_DB_ENUMERATE_NETWORKS = 1,
-- LOC_DB_ENUMERATE_ASES = 2,
-+ LOC_DB_ENUMERATE_NETWORKS = 1,
-+ LOC_DB_ENUMERATE_ASES = 2,
-+ LOC_DB_ENUMERATE_COUNTRIES = 3,
- };
-
- struct loc_database_enumerator;
-@@ -69,5 +70,7 @@ int loc_database_enumerator_next_as(
- struct loc_database_enumerator* enumerator, struct loc_as** as);
- int loc_database_enumerator_next_network(
- struct loc_database_enumerator* enumerator, struct loc_network** network);
-+int loc_database_enumerator_next_country(
-+ struct loc_database_enumerator* enumerator, struct loc_country** country);
-
- #endif
-diff --git a/src/python/database.c b/src/python/database.c
-index 581ed5b..1013a58 100644
---- a/src/python/database.c
-+++ b/src/python/database.c
-@@ -316,6 +316,10 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
- return obj;
- }
-
-+static PyObject* Database_countries(DatabaseObject* self) {
-+ return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES);
-+}
-+
- static struct PyMethodDef Database_methods[] = {
- {
- "get_as",
-@@ -364,6 +368,13 @@ static struct PyGetSetDef Database_getsetters[] = {
- NULL,
- NULL,
- },
-+ {
-+ "countries",
-+ (getter)Database_countries,
-+ NULL,
-+ NULL,
-+ NULL,
-+ },
- {
- "created_at",
- (getter)Database_get_created_at,
-@@ -462,6 +473,22 @@ static PyObject* DatabaseEnumerator_next(DatabaseEnumeratorObject* self) {
- return obj;
- }
-
-+ // Enumerate all countries
-+ struct loc_country* country = NULL;
-+
-+ r = loc_database_enumerator_next_country(self->enumerator, &country);
-+ if (r) {
-+ PyErr_SetFromErrno(PyExc_ValueError);
-+ return NULL;
-+ }
-+
-+ if (country) {
-+ PyObject* obj = new_country(&CountryType, country);
-+ loc_country_unref(country);
-+
-+ return obj;
-+ }
-+
- // Nothing found, that means the end
- PyErr_SetNone(PyExc_StopIteration);
- return NULL;
-diff --git a/src/python/location.in b/src/python/location.in
-index 7614cae..5c1effd 100644
---- a/src/python/location.in
-+++ b/src/python/location.in
-@@ -147,6 +147,18 @@ class CLI(object):
- choices=location.export.formats.keys(), default="list")
- list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags)
-
-+ # List countries
-+ list_countries = subparsers.add_parser("list-countries",
-+ help=_("Lists all countries"),
-+ )
-+ list_countries.add_argument("--show-name",
-+ action="store_true", help=_("Show the name of the country"),
-+ )
-+ list_countries.add_argument("--show-continent",
-+ action="store_true", help=_("Show the continent"),
-+ )
-+ list_countries.set_defaults(func=self.handle_list_countries)
-+
- # Export
- export = subparsers.add_parser("export",
- help=_("Exports data in many formats to load it into packet filters"),
-@@ -435,6 +447,24 @@ class CLI(object):
-
- return cls
-
-+ def handle_list_countries(self, db, ns):
-+ for country in db.countries:
-+ line = [
-+ country.code,
-+ ]
-+
-+ if ns.show_continent:
-+ line.append(country.continent_code)
-+
-+ if ns.show_name:
-+ line.append(country.name)
-+
-+ # Format the output
-+ line = " ".join(line)
-+
-+ # Print the output
-+ print(line)
-+
- def handle_list_networks_by_as(self, db, ns):
- writer = self.__get_output_formatter(ns)
-
+++ /dev/null
-commit 864dd22e17f7487a90e165274cf3f7898966028d
-Author: Michael Tremer <michael.tremer@ipfire.org>
-Date: Fri Jun 5 10:01:47 2020 +0000
-
- database: Fix broken search for networks with flags
-
- The search was ended after the first network. No matter if
- it matched, or not.
-
- Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
-
-diff --git a/src/database.c b/src/database.c
-index 8e6c5ab..fa1dad0 100644
---- a/src/database.c
-+++ b/src/database.c
-@@ -1208,6 +1208,8 @@ LOC_EXPORT int loc_database_enumerator_next_network(
- !loc_network_match_flag(*network, enumerator->flags)) {
- loc_network_unref(*network);
- *network = NULL;
-+
-+ continue;
- }
-
- return 0;
+++ /dev/null
-commit dc1df0f469668ef3dc4e1a9a7623a0ebba2b051e
-Author: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed Jun 3 17:15:27 2020 +0000
-
- downloader: Change user-agent to location
-
- Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
-
-diff --git a/src/python/downloader.py b/src/python/downloader.py
-index c9e6e00..eb28007 100644
---- a/src/python/downloader.py
-+++ b/src/python/downloader.py
-@@ -71,7 +71,7 @@ class Downloader(object):
-
- # Update headers
- headers.update({
-- "User-Agent" : "location-downloader/@VERSION@",
-+ "User-Agent" : "location/@VERSION@",
- })
-
- # Set headers
+++ /dev/null
-commit 10fa313b392a269e15bdaf316218a114d9b23b55
-Author: Michael Tremer <michael.tremer@ipfire.org>
-Date: Fri Jun 5 09:47:36 2020 +0000
-
- location(8): Export all countries by default
-
- Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
-
-diff --git a/src/python/location.in b/src/python/location.in
-index 5c1effd..6ced5f5 100644
---- a/src/python/location.in
-+++ b/src/python/location.in
-@@ -169,7 +169,7 @@ class CLI(object):
- export.add_argument("--family",
- help=_("Specify address family"), choices=("ipv6", "ipv4"),
- )
-- export.add_argument("objects", nargs="+", help=_("List country codes or ASNs to export"))
-+ export.add_argument("objects", nargs="*", help=_("List country codes or ASNs to export"))
- export.set_defaults(func=self.handle_export)
-
- args = parser.parse_args()
-@@ -539,9 +539,9 @@ class CLI(object):
- log.warning("Invalid argument: %s" % object)
- continue
-
-+ # Default to exporting all countries
- if not countries and not asns:
-- log.error("Nothing to export")
-- return 2
-+ countries = ["A1", "A2", "A3"] + [country.code for country in db.countries]
-
- # Select the output format
- writer = self.__get_output_formatter(ns)
+++ /dev/null
-commit fae36e81a32717ac43c0ce48702f6ff05b7cd029
-Author: Michael Tremer <michael.tremer@ipfire.org>
-Date: Fri Jun 5 09:57:41 2020 +0000
-
- export flagged networks with their faked country names, too
-
- This will lead to some networks showing up twice. Once with
- their real country and once with their faked one.
-
- It is likely that the first one will match.
-
- Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
-
-diff --git a/src/python/export.py b/src/python/export.py
-index 69fe964..d0bbe77 100644
---- a/src/python/export.py
-+++ b/src/python/export.py
-@@ -23,10 +23,18 @@ import logging
- import os
- import socket
-
-+import _location
-+
- # Initialise logging
- log = logging.getLogger("location.export")
- log.propagate = 1
-
-+flags = {
-+ _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1",
-+ _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2",
-+ _location.NETWORK_FLAG_ANYCAST : "A3",
-+}
-+
- class OutputWriter(object):
- suffix = "networks"
- mode = "w"
-@@ -173,6 +181,17 @@ class Exporter(object):
- except KeyError:
- pass
-
-+ # Handle flags
-+ for flag in flags:
-+ if network.has_flag(flag):
-+ # Fetch the "fake" country code
-+ country = flags[flag]
-+
-+ try:
-+ writers[country].write(network)
-+ except KeyError:
-+ pass
-+
- # Write everything to the filesystem
- for writer in writers.values():
- writer.finish()
+++ /dev/null
-commit 679e5ae2e45b98e254c651cb3102a4331c2c22eb
-Author: Michael Tremer <michael.tremer@ipfire.org>
-Date: Mon Jun 1 13:47:44 2020 +0000
-
- location-downloader: Do not change content of open database files
-
- The database might be opened by another process. When modified,
- it will return random results.
-
- Fixes: #12420
- Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
-
-diff --git a/src/python/location-downloader.in b/src/python/location-downloader.in
-index 7d06030..bf0d682 100644
---- a/src/python/location-downloader.in
-+++ b/src/python/location-downloader.in
-@@ -24,6 +24,7 @@ import lzma
- import os
- import random
- import shutil
-+import stat
- import sys
- import tempfile
- import time
-@@ -116,7 +117,7 @@ class Downloader(object):
-
- return res
-
-- def download(self, url, public_key, timestamp=None, **kwargs):
-+ def download(self, url, public_key, timestamp=None, tmpdir=None, **kwargs):
- headers = {}
-
- if timestamp:
-@@ -124,7 +125,7 @@ class Downloader(object):
- "%a, %d %b %Y %H:%M:%S GMT",
- )
-
-- t = tempfile.NamedTemporaryFile(delete=False)
-+ t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False)
- with t:
- # Try all mirrors
- for mirror in self.mirrors:
-@@ -175,6 +176,9 @@ class Downloader(object):
- t.truncate()
- continue
-
-+ # Make the file readable for everyone
-+ os.chmod(t.name, stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH)
-+
- # Return temporary file
- return t
-
-@@ -296,10 +300,13 @@ class CLI(object):
- except FileNotFoundError as e:
- db = None
-
-+ # Download the database into the correct directory
-+ tmpdir = os.path.dirname(ns.database)
-+
- # Try downloading a new database
- try:
- t = self.downloader.download("%s/%s" % (self.version, DATABASE_FILENAME),
-- public_key=ns.public_key, timestamp=timestamp)
-+ public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
-
- # If no file could be downloaded, log a message
- except FileNotFoundError as e:
-@@ -310,11 +317,8 @@ class CLI(object):
- if not t:
- return 3
-
-- # Write temporary file to destination
-- shutil.copyfile(t.name, ns.database)
--
-- # Remove temporary file
-- os.unlink(t.name)
-+ # Move temporary file to destination
-+ shutil.move(t.name, ns.database)
-
- return 0
-
+++ /dev/null
-commit 141b10999b280b2563580c705d5d23dc4c442deb
-Author: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed Jun 3 16:31:44 2020 +0000
-
- location-exporter: Do not mistake country AS for an AS number
-
- Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
-
-diff --git a/src/python/location-exporter.in b/src/python/location-exporter.in
-index 894bb44..5454561 100644
---- a/src/python/location-exporter.in
-+++ b/src/python/location-exporter.in
-@@ -22,6 +22,7 @@ import io
- import ipaddress
- import logging
- import os.path
-+import re
- import socket
- import sys
-
-@@ -258,12 +259,9 @@ class CLI(object):
- families = [ socket.AF_INET6, socket.AF_INET ]
-
- for object in ns.objects:
-- if object.startswith("AS"):
-- try:
-- object = int(object[2:])
-- except ValueError:
-- log.error("Invalid argument: %s" % object)
-- return 2
-+ m = re.match("^AS(\d+)$", object)
-+ if m:
-+ object = int(m.group(1))
-
- asns.append(object)
-
+++ /dev/null
-commit 92af07adfb1e06fe1b055fbcf5ba61159637cd73
-Author: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed Jun 3 16:33:44 2020 +0000
-
- location-exporter: Warn, but do not fail on invalid input
-
- Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
-
-diff --git a/src/python/location-exporter.in b/src/python/location-exporter.in
-index 5454561..d82f1d3 100644
---- a/src/python/location-exporter.in
-+++ b/src/python/location-exporter.in
-@@ -270,8 +270,12 @@ class CLI(object):
- countries.append(object)
-
- else:
-- log.error("Invalid argument: %s" % object)
-- return 2
-+ log.warning("Invalid argument: %s" % object)
-+ continue
-+
-+ if not countries and not asns:
-+ log.error("Nothing to export")
-+ return 2
-
- # Open the database
- try:
+++ /dev/null
-commit 228d0e74ec47c9954d3a0e1da2e1c0fc6c1b518f
-Author: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed Jun 3 16:15:24 2020 +0000
-
- location-query: Require at least one flag
-
- Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
-
-diff --git a/src/python/location-query.in b/src/python/location-query.in
-index 5f05b5c..dfdff8c 100644
---- a/src/python/location-query.in
-+++ b/src/python/location-query.in
-@@ -246,7 +246,13 @@ class CLI(object):
- args.family = 0
-
- # Call function
-- ret = args.func(db, args)
-+ try:
-+ ret = args.func(db, args)
-+
-+ # Catch invalid inputs
-+ except ValueError as e:
-+ sys.stderr.write("%s\n" % e)
-+ ret = 2
-
- # Return with exit code
- if ret:
-@@ -451,6 +457,9 @@ class CLI(object):
- if ns.anycast:
- flags |= location.NETWORK_FLAG_ANYCAST
-
-+ if not flags:
-+ raise ValueError(_("You must at least pass one flag"))
-+
- with self.__get_output_formatter(ns) as f:
- for n in db.search_networks(flags=flags, family=ns.family):
- f.network(n)
+++ /dev/null
-commit 889b932aa6172c96872be545af37d351f7c1c705
-Author: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed Jun 3 17:10:35 2020 +0000
-
- location-downloader: Merge man page into location-query
-
- Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
-
-diff --git a/Makefile.am b/Makefile.am
-index c0b1300..91f0436 100644
---- a/Makefile.am
-+++ b/Makefile.am
-@@ -37,6 +37,9 @@ LIBLOC_CURRENT=0
- LIBLOC_REVISION=0
- LIBLOC_AGE=0
-
-+pythondir = $(prefix)/lib/python3/dist-packages
-+pyexecdir = $(prefix)/lib/python$(PYTHON_VERSION)/lib-dynload
-+
- DISTCHECK_CONFIGURE_FLAGS = \
- --with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir)
-
-@@ -371,7 +374,6 @@ src_test_signature_LDADD = \
- # ------------------------------------------------------------------------------
-
- MANPAGES = \
-- man/location-downloader.8 \
- man/location-query.8
-
- MANPAGES_TXT = $(patsubst %.8,%.txt,$(MANPAGES))
-diff --git a/man/location-downloader.txt b/man/location-downloader.txt
-deleted file mode 100644
-index d733923..0000000
---- a/man/location-downloader.txt
-+++ /dev/null
-@@ -1,61 +0,0 @@
--= location-downloader(8)
--
--== NAME
--location-downloader - Download a location database
--
--== SYNOPSIS
--[verse]
--`location-downloader update`
--
--== DESCRIPTION
--The `location-downloader` command updates the local version of the
--location database.
--
--== OPTIONS
--
----database FILE::
---d FILE::
-- The path of the database which is being updated.
-- +
-- If this option is omitted, the system's database will be opened.
--
----quiet::
-- Enable quiet mode
--
----debug::
-- Enable debugging mode
--
--== COMMANDS
--
--'update'::
-- This command will try to update the local database.
-- +
-- It will terminate with a return code of zero if the database has been
-- successfully updated. 1 on error, 2 on invalid call and 3 if the
-- database was already the latest version.
--
--'verify'::
-- Verifies the downloaded database.
--
--'--help'::
-- Shows a short help text on using this program.
--
--'--version'::
-- Shows the program's version and exists.
--
--== EXIT CODES
--The 'location-downloader' command will normally exit with code zero.
--If there has been a problem and the requested action could not be performed,
--the exit code is unequal to zero.
--
--== HOW IT WORKS
--The downloader checks a DNS record for the latest version of the database.
--It will then try to download a file with that version from a mirror server.
--If the downloaded file is outdated, the next mirror will be tried until we
--have found a file that is recent enough.
--
--== BUGS
--Please report all bugs to the bugtracker at https://bugzilla.ipfire.org/.
--
--== AUTHORS
--Michael Tremer
-diff --git a/man/location-query.txt b/man/location-query.txt
-index b91e8e1..acb43cd 100644
---- a/man/location-query.txt
-+++ b/man/location-query.txt
-@@ -8,6 +8,8 @@ location-query - Query the location database
- `location-query lookup ADDRESS [ADDRESS...]`
- `location-query get-as ASN [ASN...]`
- `location-query search-as STRING`
-+`location-query update`
-+`location-query verify`
- `location-query list-networks-by-as ASN`
- `location-query list-networks-by-cc COUNTRY_CODE`
- `location-query list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]`
-@@ -47,6 +49,16 @@ or countries.
- +
- The search will be performed case-insensitively.
-
-+'update'::
-+ This command will try to update the local database.
-+ +
-+ It will terminate with a return code of zero if the database has been
-+ successfully updated. 1 on error, 2 on invalid call and 3 if the
-+ database was already the latest version.
-+
-+'verify'::
-+ Verifies the downloaded database.
-+
- 'list-networks-by-as [--family=[ipv6|ipv4]]Â [--output-format FORMAT]Â ASN'::
- Lists all networks which belong to this Autonomous System.
- +
-@@ -85,6 +97,12 @@ The 'location-query' command will normally exit with code zero.
- If there has been a problem and the requested action could not be performed,
- the exit code is unequal to zero.
-
-+== HOW IT WORKS
-+The downloader checks a DNS record for the latest version of the database.
-+It will then try to download a file with that version from a mirror server.
-+If the downloaded file is outdated, the next mirror will be tried until we
-+have found a file that is recent enough.
-+
- == BUGS
- Please report all bugs to the bugtracker at https://bugzilla.ipfire.org/.
-
+++ /dev/null
-commit 88ef7e9cd4b3a1a5662c7dc071bd7a44e1242cba
-Author: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed Jun 3 18:36:28 2020 +0000
-
- Merge location-exporter(8) into location(8)
-
- Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
-
-diff --git a/Makefile.am b/Makefile.am
-index 59870b1..9f520cc 100644
---- a/Makefile.am
-+++ b/Makefile.am
-@@ -150,6 +150,7 @@ dist_pkgpython_PYTHON = \
- src/python/__init__.py \
- src/python/database.py \
- src/python/downloader.py \
-+ src/python/export.py \
- src/python/i18n.py \
- src/python/importer.py \
- src/python/logger.py
-@@ -239,17 +240,14 @@ uninstall-perl:
-
- bin_SCRIPTS = \
- src/python/location \
-- src/python/location-exporter \
- src/python/location-importer
-
- EXTRA_DIST += \
- src/python/location.in \
-- src/python/location-exporter.in \
- src/python/location-importer.in
-
- CLEANFILES += \
- src/python/location \
-- src/python/location-exporter \
- src/python/location-importer
-
- # ------------------------------------------------------------------------------
-diff --git a/src/python/export.py b/src/python/export.py
-new file mode 100644
-index 0000000..69fe964
---- /dev/null
-+++ b/src/python/export.py
-@@ -0,0 +1,185 @@
-+#!/usr/bin/python3
-+###############################################################################
-+# #
-+# libloc - A library to determine the location of someone on the Internet #
-+# #
-+# Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
-+# #
-+# This library is free software; you can redistribute it and/or #
-+# modify it under the terms of the GNU Lesser General Public #
-+# License as published by the Free Software Foundation; either #
-+# version 2.1 of the License, or (at your option) any later version. #
-+# #
-+# This library is distributed in the hope that it will be useful, #
-+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
-+# Lesser General Public License for more details. #
-+# #
-+###############################################################################
-+
-+import io
-+import ipaddress
-+import logging
-+import os
-+import socket
-+
-+# Initialise logging
-+log = logging.getLogger("location.export")
-+log.propagate = 1
-+
-+class OutputWriter(object):
-+ suffix = "networks"
-+ mode = "w"
-+
-+ def __init__(self, f, prefix=None):
-+ self.f, self.prefix = f, prefix
-+
-+ # Immediately write the header
-+ self._write_header()
-+
-+ @classmethod
-+ def open(cls, filename, **kwargs):
-+ """
-+ Convenience function to open a file
-+ """
-+ f = open(filename, cls.mode)
-+
-+ return cls(f, **kwargs)
-+
-+ def __repr__(self):
-+ return "<%s f=%s>" % (self.__class__.__name__, self.f)
-+
-+ def _write_header(self):
-+ """
-+ The header of the file
-+ """
-+ pass
-+
-+ def _write_footer(self):
-+ """
-+ The footer of the file
-+ """
-+ pass
-+
-+ def write(self, network):
-+ self.f.write("%s\n" % network)
-+
-+ def finish(self):
-+ """
-+ Called when all data has been written
-+ """
-+ self._write_footer()
-+
-+ # Close the file
-+ self.f.close()
-+
-+
-+class IpsetOutputWriter(OutputWriter):
-+ """
-+ For ipset
-+ """
-+ suffix = "ipset"
-+
-+ def _write_header(self):
-+ self.f.write("create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.prefix)
-+
-+ def write(self, network):
-+ self.f.write("add %s %s\n" % (self.prefix, network))
-+
-+
-+class NftablesOutputWriter(OutputWriter):
-+ """
-+ For nftables
-+ """
-+ suffix = "set"
-+
-+ def _write_header(self):
-+ self.f.write("define %s = {\n" % self.prefix)
-+
-+ def _write_footer(self):
-+ self.f.write("}\n")
-+
-+ def write(self, network):
-+ self.f.write(" %s,\n" % network)
-+
-+
-+class XTGeoIPOutputWriter(OutputWriter):
-+ """
-+ Formats the output in that way, that it can be loaded by
-+ the xt_geoip kernel module from xtables-addons.
-+ """
-+ suffix = "iv"
-+ mode = "wb"
-+
-+ def write(self, network):
-+ n = ipaddress.ip_network("%s" % network)
-+
-+ for address in (n.network_address, n.broadcast_address):
-+ bytes = socket.inet_pton(
-+ socket.AF_INET6 if address.version == 6 else socket.AF_INET,
-+ "%s" % address,
-+ )
-+
-+ self.f.write(bytes)
-+
-+
-+formats = {
-+ "ipset" : IpsetOutputWriter,
-+ "list" : OutputWriter,
-+ "nftables" : NftablesOutputWriter,
-+ "xt_geoip" : XTGeoIPOutputWriter,
-+}
-+
-+class Exporter(object):
-+ def __init__(self, db, writer):
-+ self.db, self.writer = db, writer
-+
-+ def export(self, directory, families, countries, asns):
-+ for family in families:
-+ log.debug("Exporting family %s" % family)
-+
-+ writers = {}
-+
-+ # Create writers for countries
-+ for country_code in countries:
-+ filename = self._make_filename(
-+ directory, prefix=country_code, suffix=self.writer.suffix, family=family,
-+ )
-+
-+ writers[country_code] = self.writer.open(filename, prefix="CC_%s" % country_code)
-+
-+ # Create writers for ASNs
-+ for asn in asns:
-+ filename = self._make_filename(
-+ directory, "AS%s" % asn, suffix=self.writer.suffix, family=family,
-+ )
-+
-+ writers[asn] = self.writer.open(filename, prefix="AS%s" % asn)
-+
-+ # Get all networks that match the family
-+ networks = self.db.search_networks(family=family)
-+
-+ # Walk through all networks
-+ for network in networks:
-+ # Write matching countries
-+ try:
-+ writers[network.country_code].write(network)
-+ except KeyError:
-+ pass
-+
-+ # Write matching ASNs
-+ try:
-+ writers[network.asn].write(network)
-+ except KeyError:
-+ pass
-+
-+ # Write everything to the filesystem
-+ for writer in writers.values():
-+ writer.finish()
-+
-+ def _make_filename(self, directory, prefix, suffix, family):
-+ filename = "%s.%s%s" % (
-+ prefix, suffix, "6" if family == socket.AF_INET6 else "4"
-+ )
-+
-+ return os.path.join(directory, filename)
-diff --git a/src/python/location-exporter.in b/src/python/location-exporter.in
-deleted file mode 100644
-index d82f1d3..0000000
---- a/src/python/location-exporter.in
-+++ /dev/null
-@@ -1,300 +0,0 @@
--#!/usr/bin/python3
--###############################################################################
--# #
--# libloc - A library to determine the location of someone on the Internet #
--# #
--# Copyright (C) 2019 IPFire Development Team <info@ipfire.org> #
--# #
--# This library is free software; you can redistribute it and/or #
--# modify it under the terms of the GNU Lesser General Public #
--# License as published by the Free Software Foundation; either #
--# version 2.1 of the License, or (at your option) any later version. #
--# #
--# This library is distributed in the hope that it will be useful, #
--# but WITHOUT ANY WARRANTY; without even the implied warranty of #
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
--# Lesser General Public License for more details. #
--# #
--###############################################################################
--
--import argparse
--import io
--import ipaddress
--import logging
--import os.path
--import re
--import socket
--import sys
--
--# Load our location module
--import location
--from location.i18n import _
--
--# Initialise logging
--log = logging.getLogger("location.exporter")
--log.propagate = 1
--
--class OutputWriter(object):
-- suffix = "networks"
--
-- def __init__(self, family, country_code=None, asn=None):
-- self.family, self.country_code, self.asn = family, country_code, asn
--
-- self.f = io.BytesIO()
--
-- def write_out(self, directory):
-- # Make the output filename
-- filename = os.path.join(
-- directory, self._make_filename(),
-- )
--
-- with open(filename, "wb") as f:
-- self._write_header(f)
--
-- # Copy all data into the file
-- f.write(self.f.getbuffer())
--
-- self._write_footer(f)
--
-- def _make_filename(self):
-- return "%s.%s%s" % (
-- self.country_code or "AS%s" % self.asn,
-- self.suffix,
-- "6" if self.family == socket.AF_INET6 else "4"
-- )
--
-- @property
-- def name(self):
-- if self.country_code:
-- return "CC_%s" % self.country_code
--
-- if self.asn:
-- return "AS%s" % self.asn
--
-- def _write_header(self, f):
-- """
-- The header of the file
-- """
-- pass
--
-- def _write_footer(self, f):
-- """
-- The footer of the file
-- """
-- pass
--
-- def write(self, network):
-- s = "%s\n" % network
--
-- self.f.write(s.encode("ascii"))
--
--
--class IpsetOutputWriter(OutputWriter):
-- """
-- For ipset
-- """
-- suffix = "ipset"
--
-- def _write_header(self, f):
-- h = "create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.name
--
-- f.write(h.encode("ascii"))
--
-- def write(self, network):
-- s = "add %s %s\n" % (self.name, network)
--
-- self.f.write(s.encode("ascii"))
--
--
--class NftablesOutputWriter(OutputWriter):
-- """
-- For nftables
-- """
-- suffix = "set"
--
-- def _write_header(self, f):
-- h = "define %s = {\n" % self.name
--
-- f.write(h.encode("ascii"))
--
-- def _write_footer(self, f):
-- f.write(b"}")
--
-- def write(self, network):
-- s = " %s,\n" % network
--
-- self.f.write(s.encode("ascii"))
--
--
--class XTGeoIPOutputWriter(OutputWriter):
-- """
-- Formats the output in that way, that it can be loaded by
-- the xt_geoip kernel module from xtables-addons.
-- """
-- suffix = "iv"
--
-- def write(self, network):
-- n = ipaddress.ip_network("%s" % network)
--
-- for address in (n.network_address, n.broadcast_address):
-- bytes = socket.inet_pton(
-- socket.AF_INET6 if address.version == 6 else socket.AF_INET,
-- "%s" % address,
-- )
--
-- self.f.write(bytes)
--
--
--class Exporter(object):
-- def __init__(self, db, writer):
-- self.db = db
-- self.writer = writer
--
-- def export(self, directory, families, countries, asns):
-- for family in families:
-- log.debug("Exporting family %s" % family)
--
-- writers = {}
--
-- # Create writers for countries
-- for country_code in countries:
-- writers[country_code] = self.writer(family, country_code=country_code)
--
-- # Create writers for ASNs
-- for asn in asns:
-- writers[asn] = self.writer(family, asn=asn)
--
-- # Get all networks that match the family
-- networks = self.db.search_networks(family=family)
--
-- # Walk through all networks
-- for network in networks:
-- # Write matching countries
-- if network.country_code in countries:
-- writers[network.country_code].write(network)
--
-- # Write matching ASNs
-- if network.asn in asns:
-- writers[network.asn].write(network)
--
-- # Write everything to the filesystem
-- for writer in writers.values():
-- writer.write_out(directory)
--
--
--class CLI(object):
-- output_formats = {
-- "ipset" : IpsetOutputWriter,
-- "list" : OutputWriter,
-- "nftables" : NftablesOutputWriter,
-- "xt_geoip" : XTGeoIPOutputWriter,
-- }
--
-- def parse_cli(self):
-- parser = argparse.ArgumentParser(
-- description=_("Location Exporter Command Line Interface"),
-- )
--
-- # Global configuration flags
-- parser.add_argument("--debug", action="store_true",
-- help=_("Enable debug output"))
-- parser.add_argument("--quiet", action="store_true",
-- help=_("Enable quiet mode"))
--
-- # version
-- parser.add_argument("--version", action="version",
-- version="%(prog)s @VERSION@")
--
-- # database
-- parser.add_argument("--database", "-d",
-- default="@databasedir@/database.db", help=_("Path to database"),
-- )
--
-- # format
-- parser.add_argument("--format", help=_("Output format"),
-- default="list", choices=self.output_formats.keys())
--
-- # directory
-- parser.add_argument("--directory", help=_("Output directory"), required=True)
--
-- # family
-- parser.add_argument("--family", help=_("Specify address family"), choices=("ipv6", "ipv4"))
--
-- # Countries and Autonomous Systems
-- parser.add_argument("objects", nargs="+")
--
-- args = parser.parse_args()
--
-- # Configure logging
-- if args.debug:
-- location.logger.set_level(logging.DEBUG)
-- elif args.quiet:
-- location.logger.set_level(logging.WARNING)
--
-- return args
--
-- def run(self):
-- # Parse command line arguments
-- args = self.parse_cli()
--
-- # Call function
-- ret = self.handle_export(args)
--
-- # Return with exit code
-- if ret:
-- sys.exit(ret)
--
-- # Otherwise just exit
-- sys.exit(0)
--
-- def handle_export(self, ns):
-- countries, asns = [], []
--
-- # Translate family
-- if ns.family == "ipv6":
-- families = [ socket.AF_INET6 ]
-- elif ns.family == "ipv4":
-- families = [ socket.AF_INET ]
-- else:
-- families = [ socket.AF_INET6, socket.AF_INET ]
--
-- for object in ns.objects:
-- m = re.match("^AS(\d+)$", object)
-- if m:
-- object = int(m.group(1))
--
-- asns.append(object)
--
-- elif location.country_code_is_valid(object) \
-- or object in ("A1", "A2", "A3"):
-- countries.append(object)
--
-- else:
-- log.warning("Invalid argument: %s" % object)
-- continue
--
-- if not countries and not asns:
-- log.error("Nothing to export")
-- return 2
--
-- # Open the database
-- try:
-- db = location.Database(ns.database)
-- except FileNotFoundError as e:
-- log.error("Count not open database: %s" % ns.database)
-- return 1
--
-- # Select the output format
-- writer = self.output_formats.get(ns.format)
-- assert writer
--
-- e = Exporter(db, writer)
-- e.export(ns.directory, countries=countries, asns=asns, families=families)
--
--
--def main():
-- # Run the command line interface
-- c = CLI()
-- c.run()
--
--main()
-diff --git a/src/python/location.in b/src/python/location.in
-index 10618e2..7614cae 100644
---- a/src/python/location.in
-+++ b/src/python/location.in
-@@ -22,6 +22,7 @@ import datetime
- import ipaddress
- import logging
- import os
-+import re
- import shutil
- import socket
- import sys
-@@ -30,6 +31,8 @@ import time
- # Load our location module
- import location
- import location.downloader
-+import location.export
-+
- from location.i18n import _
-
- # Setup logging
-@@ -37,88 +40,7 @@ log = logging.getLogger("location")
-
- # Output formatters
-
--class OutputFormatter(object):
-- def __init__(self, ns):
-- self.ns = ns
--
-- def __enter__(self):
-- # Open the output
-- self.open()
--
-- return self
--
-- def __exit__(self, type, value, tb):
-- if tb is None:
-- self.close()
--
-- @property
-- def name(self):
-- if "country_code" in self.ns:
-- return "networks_country_%s" % self.ns.country_code[0]
--
-- elif "asn" in self.ns:
-- return "networks_AS%s" % self.ns.asn[0]
--
-- def open(self):
-- pass
--
-- def close(self):
-- pass
--
-- def network(self, network):
-- print(network)
--
--
--class IpsetOutputFormatter(OutputFormatter):
-- """
-- For nftables
-- """
-- def open(self):
-- print("create %s hash:net family inet hashsize 1024 maxelem 65536" % self.name)
--
-- def network(self, network):
-- print("add %s %s" % (self.name, network))
--
--
--class NftablesOutputFormatter(OutputFormatter):
-- """
-- For nftables
-- """
-- def open(self):
-- print("define %s = {" % self.name)
--
-- def close(self):
-- print("}")
--
-- def network(self, network):
-- print(" %s," % network)
--
--
--class XTGeoIPOutputFormatter(OutputFormatter):
-- """
-- Formats the output in that way, that it can be loaded by
-- the xt_geoip kernel module from xtables-addons.
-- """
-- def network(self, network):
-- n = ipaddress.ip_network("%s" % network)
--
-- for address in (n.network_address, n.broadcast_address):
-- bytes = socket.inet_pton(
-- socket.AF_INET6 if address.version == 6 else socket.AF_INET,
-- "%s" % address,
-- )
--
-- os.write(1, bytes)
--
--
- class CLI(object):
-- output_formats = {
-- "ipset" : IpsetOutputFormatter,
-- "list" : OutputFormatter,
-- "nftables" : NftablesOutputFormatter,
-- "xt_geoip" : XTGeoIPOutputFormatter,
-- }
--
- def parse_cli(self):
- parser = argparse.ArgumentParser(
- description=_("Location Database Command Line Interface"),
-@@ -193,8 +115,8 @@ class CLI(object):
- )
- list_networks_by_as.add_argument("asn", nargs=1, type=int)
- list_networks_by_as.add_argument("--family", choices=("ipv6", "ipv4"))
-- list_networks_by_as.add_argument("--output-format",
-- choices=self.output_formats.keys(), default="list")
-+ list_networks_by_as.add_argument("--format",
-+ choices=location.export.formats.keys(), default="list")
- list_networks_by_as.set_defaults(func=self.handle_list_networks_by_as)
-
- # List all networks in a country
-@@ -203,8 +125,8 @@ class CLI(object):
- )
- list_networks_by_cc.add_argument("country_code", nargs=1)
- list_networks_by_cc.add_argument("--family", choices=("ipv6", "ipv4"))
-- list_networks_by_cc.add_argument("--output-format",
-- choices=self.output_formats.keys(), default="list")
-+ list_networks_by_cc.add_argument("--format",
-+ choices=location.export.formats.keys(), default="list")
- list_networks_by_cc.set_defaults(func=self.handle_list_networks_by_cc)
-
- # List all networks with flags
-@@ -221,10 +143,23 @@ class CLI(object):
- action="store_true", help=_("Anycasts"),
- )
- list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4"))
-- list_networks_by_flags.add_argument("--output-format",
-- choices=self.output_formats.keys(), default="list")
-+ list_networks_by_flags.add_argument("--format",
-+ choices=location.export.formats.keys(), default="list")
- list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags)
-
-+ # Export
-+ export = subparsers.add_parser("export",
-+ help=_("Exports data in many formats to load it into packet filters"),
-+ )
-+ export.add_argument("--format", help=_("Output format"),
-+ choices=location.export.formats.keys(), default="list")
-+ export.add_argument("--directory", help=_("Output directory"), required=True)
-+ export.add_argument("--family",
-+ help=_("Specify address family"), choices=("ipv6", "ipv4"),
-+ )
-+ export.add_argument("objects", nargs="+", help=_("List country codes or ASNs to export"))
-+ export.set_defaults(func=self.handle_export)
-+
- args = parser.parse_args()
-
- # Configure logging
-@@ -494,25 +429,36 @@ class CLI(object):
-
- def __get_output_formatter(self, ns):
- try:
-- cls = self.output_formats[ns.output_format]
-+ cls = location.export.formats[ns.format]
- except KeyError:
-- cls = OutputFormatter
-+ cls = location.export.OutputFormatter
-
-- return cls(ns)
-+ return cls
-
- def handle_list_networks_by_as(self, db, ns):
-- with self.__get_output_formatter(ns) as f:
-- for asn in ns.asn:
-- # Print all matching networks
-- for n in db.search_networks(asn=asn, family=ns.family):
-- f.network(n)
-+ writer = self.__get_output_formatter(ns)
-+
-+ for asn in ns.asn:
-+ f = writer(sys.stdout, prefix="AS%s" % asn)
-+
-+ # Print all matching networks
-+ for n in db.search_networks(asn=asn, family=ns.family):
-+ f.write(n)
-+
-+ f.finish()
-
- def handle_list_networks_by_cc(self, db, ns):
-- with self.__get_output_formatter(ns) as f:
-- for country_code in ns.country_code:
-- # Print all matching networks
-- for n in db.search_networks(country_code=country_code, family=ns.family):
-- f.network(n)
-+ writer = self.__get_output_formatter(ns)
-+
-+ for country_code in ns.country_code:
-+ # Open standard output
-+ f = writer(sys.stdout, prefix=country_code)
-+
-+ # Print all matching networks
-+ for n in db.search_networks(country_code=country_code, family=ns.family):
-+ f.write(n)
-+
-+ f.finish()
-
- def handle_list_networks_by_flags(self, db, ns):
- flags = 0
-@@ -529,9 +475,49 @@ class CLI(object):
- if not flags:
- raise ValueError(_("You must at least pass one flag"))
-
-- with self.__get_output_formatter(ns) as f:
-- for n in db.search_networks(flags=flags, family=ns.family):
-- f.network(n)
-+ writer = self.__get_output_formatter(ns)
-+ f = writer(sys.stdout, prefix="custom")
-+
-+ for n in db.search_networks(flags=flags, family=ns.family):
-+ f.write(n)
-+
-+ f.finish()
-+
-+ def handle_export(self, db, ns):
-+ countries, asns = [], []
-+
-+ # Translate family
-+ if ns.family == "ipv6":
-+ families = [ socket.AF_INET6 ]
-+ elif ns.family == "ipv4":
-+ families = [ socket.AF_INET ]
-+ else:
-+ families = [ socket.AF_INET6, socket.AF_INET ]
-+
-+ for object in ns.objects:
-+ m = re.match("^AS(\d+)$", object)
-+ if m:
-+ object = int(m.group(1))
-+
-+ asns.append(object)
-+
-+ elif location.country_code_is_valid(object) \
-+ or object in ("A1", "A2", "A3"):
-+ countries.append(object)
-+
-+ else:
-+ log.warning("Invalid argument: %s" % object)
-+ continue
-+
-+ if not countries and not asns:
-+ log.error("Nothing to export")
-+ return 2
-+
-+ # Select the output format
-+ writer = self.__get_output_formatter(ns)
-+
-+ e = location.export.Exporter(db, writer)
-+ e.export(ns.directory, countries=countries, asns=asns, families=families)
-
-
- def main():
+++ /dev/null
-commit a6f1e3463d4c2085c203ad58072d7a154b663904
-Author: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed Jun 3 17:06:13 2020 +0000
-
- Move location-downloader functionality into location-query
-
- The commands are very long and confusion. Hence we merge this
- all into one command.
-
- Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
-
-diff --git a/Makefile.am b/Makefile.am
-index 31869e0..c0b1300 100644
---- a/Makefile.am
-+++ b/Makefile.am
-@@ -146,6 +146,7 @@ CLEANFILES += \
- dist_pkgpython_PYTHON = \
- src/python/__init__.py \
- src/python/database.py \
-+ src/python/downloader.py \
- src/python/i18n.py \
- src/python/importer.py \
- src/python/logger.py
-@@ -234,19 +235,16 @@ uninstall-perl:
- $(DESTDIR)/$(prefix)/man/man3/Location.3pm
-
- bin_SCRIPTS = \
-- src/python/location-downloader \
- src/python/location-exporter \
- src/python/location-importer \
- src/python/location-query
-
- EXTRA_DIST += \
-- src/python/location-downloader.in \
- src/python/location-exporter.in \
- src/python/location-importer.in \
- src/python/location-query.in
-
- CLEANFILES += \
-- src/python/location-downloader \
- src/python/location-exporter \
- src/python/location-importer \
- src/python/location-query
-diff --git a/src/python/location-downloader.in b/src/python/downloader.py
-similarity index 60%
-rename from src/python/location-downloader.in
-rename to src/python/downloader.py
-index bf0d682..c9e6e00 100644
---- a/src/python/location-downloader.in
-+++ b/src/python/downloader.py
-@@ -3,7 +3,7 @@
- # #
- # libloc - A library to determine the location of someone on the Internet #
- # #
--# Copyright (C) 2019 IPFire Development Team <info@ipfire.org> #
-+# Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
- # #
- # This library is free software; you can redistribute it and/or #
- # modify it under the terms of the GNU Lesser General Public #
-@@ -17,24 +17,18 @@
- # #
- ###############################################################################
-
--import argparse
--import datetime
- import logging
- import lzma
- import os
- import random
--import shutil
- import stat
--import sys
- import tempfile
- import time
- import urllib.error
- import urllib.parse
- import urllib.request
-
--# Load our location module
--import location
--from location.i18n import _
-+from _location import Database, DATABASE_VERSION_LATEST
-
- DATABASE_FILENAME = "location.db.xz"
- MIRRORS = (
-@@ -46,9 +40,11 @@ log = logging.getLogger("location.downloader")
- log.propagate = 1
-
- class Downloader(object):
-- def __init__(self, version, mirrors):
-+ def __init__(self, version=DATABASE_VERSION_LATEST, mirrors=None):
- self.version = version
-- self.mirrors = list(mirrors)
-+
-+ # Set mirrors or use defaults
-+ self.mirrors = list(mirrors or MIRRORS)
-
- # Randomize mirrors
- random.shuffle(self.mirrors)
-@@ -117,9 +113,10 @@ class Downloader(object):
-
- return res
-
-- def download(self, url, public_key, timestamp=None, tmpdir=None, **kwargs):
-- headers = {}
-+ def download(self, public_key, timestamp=None, tmpdir=None, **kwargs):
-+ url = "%s/%s" % (self.version, DATABASE_FILENAME)
-
-+ headers = {}
- if timestamp:
- headers["If-Modified-Since"] = timestamp.strftime(
- "%a, %d %b %Y %H:%M:%S GMT",
-@@ -191,7 +188,7 @@ class Downloader(object):
- """
- log.debug("Opening downloaded database at %s" % f.name)
-
-- db = location.Database(f.name)
-+ db = Database(f.name)
-
- # Database is not recent
- if timestamp and db.created_at < timestamp.timestamp():
-@@ -208,141 +205,3 @@ class Downloader(object):
- return False
-
- return True
--
--
--class CLI(object):
-- def __init__(self):
-- # Which version are we downloading?
-- self.version = location.DATABASE_VERSION_LATEST
--
-- self.downloader = Downloader(version=self.version, mirrors=MIRRORS)
--
-- def parse_cli(self):
-- parser = argparse.ArgumentParser(
-- description=_("Location Downloader Command Line Interface"),
-- )
-- subparsers = parser.add_subparsers()
--
-- # Global configuration flags
-- parser.add_argument("--debug", action="store_true",
-- help=_("Enable debug output"))
-- parser.add_argument("--quiet", action="store_true",
-- help=_("Enable quiet mode"))
--
-- # version
-- parser.add_argument("--version", action="version",
-- version="%(prog)s @VERSION@")
--
-- # database
-- parser.add_argument("--database", "-d",
-- default="@databasedir@/database.db", help=_("Path to database"),
-- )
--
-- # public key
-- parser.add_argument("--public-key", "-k",
-- default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
-- )
--
-- # Update
-- update = subparsers.add_parser("update", help=_("Update database"))
-- update.set_defaults(func=self.handle_update)
--
-- # Verify
-- verify = subparsers.add_parser("verify",
-- help=_("Verify the downloaded database"))
-- verify.set_defaults(func=self.handle_verify)
--
-- args = parser.parse_args()
--
-- # Configure logging
-- if args.debug:
-- location.logger.set_level(logging.DEBUG)
-- elif args.quiet:
-- location.logger.set_level(logging.WARNING)
--
-- # Print usage if no action was given
-- if not "func" in args:
-- parser.print_usage()
-- sys.exit(2)
--
-- return args
--
-- def run(self):
-- # Parse command line arguments
-- args = self.parse_cli()
--
-- # Call function
-- ret = args.func(args)
--
-- # Return with exit code
-- if ret:
-- sys.exit(ret)
--
-- # Otherwise just exit
-- sys.exit(0)
--
-- def handle_update(self, ns):
-- # Fetch the timestamp we need from DNS
-- t = location.discover_latest_version(self.version)
--
-- # Parse timestamp into datetime format
-- timestamp = datetime.datetime.fromtimestamp(t) if t else None
--
-- # Open database
-- try:
-- db = location.Database(ns.database)
--
-- # Check if we are already on the latest version
-- if timestamp and db.created_at >= timestamp.timestamp():
-- log.info("Already on the latest version")
-- return
--
-- except FileNotFoundError as e:
-- db = None
--
-- # Download the database into the correct directory
-- tmpdir = os.path.dirname(ns.database)
--
-- # Try downloading a new database
-- try:
-- t = self.downloader.download("%s/%s" % (self.version, DATABASE_FILENAME),
-- public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
--
-- # If no file could be downloaded, log a message
-- except FileNotFoundError as e:
-- log.error("Could not download a new database")
-- return 1
--
-- # If we have not received a new file, there is nothing to do
-- if not t:
-- return 3
--
-- # Move temporary file to destination
-- shutil.move(t.name, ns.database)
--
-- return 0
--
-- def handle_verify(self, ns):
-- try:
-- db = location.Database(ns.database)
-- except FileNotFoundError as e:
-- log.error("%s: %s" % (ns.database, e))
-- return 127
--
-- # Verify the database
-- with open(ns.public_key, "r") as f:
-- if not db.verify(f):
-- log.error("Could not verify database")
-- return 1
--
-- # Success
-- log.debug("Database successfully verified")
-- return 0
--
--
--def main():
-- # Run the command line interface
-- c = CLI()
-- c.run()
--
--main()
-diff --git a/src/python/location-query.in b/src/python/location-query.in
-index dfdff8c..0291786 100644
---- a/src/python/location-query.in
-+++ b/src/python/location-query.in
-@@ -18,16 +18,23 @@
- ###############################################################################
-
- import argparse
-+import datetime
- import ipaddress
-+import logging
- import os
-+import shutil
- import socket
- import sys
- import time
-
- # Load our location module
- import location
-+import location.downloader
- from location.i18n import _
-
-+# Setup logging
-+log = logging.getLogger("location")
-+
- # Output formatters
-
- class OutputFormatter(object):
-@@ -157,6 +164,15 @@ class CLI(object):
- dump.add_argument("output", nargs="?", type=argparse.FileType("w"))
- dump.set_defaults(func=self.handle_dump)
-
-+ # Update
-+ update = subparsers.add_parser("update", help=_("Update database"))
-+ update.set_defaults(func=self.handle_update)
-+
-+ # Verify
-+ verify = subparsers.add_parser("verify",
-+ help=_("Verify the downloaded database"))
-+ verify.set_defaults(func=self.handle_verify)
-+
- # Get AS
- get_as = subparsers.add_parser("get-as",
- help=_("Get information about one or multiple Autonomous Systems"),
-@@ -423,6 +439,59 @@ class CLI(object):
- for a in db.search_as(query):
- print(a)
-
-+ def handle_update(self, db, ns):
-+ # Fetch the timestamp we need from DNS
-+ t = location.discover_latest_version()
-+
-+ # Parse timestamp into datetime format
-+ timestamp = datetime.datetime.fromtimestamp(t) if t else None
-+
-+ # Check the version of the local database
-+ if db and timestamp and db.created_at >= timestamp.timestamp():
-+ log.info("Already on the latest version")
-+ return
-+
-+ # Download the database into the correct directory
-+ tmpdir = os.path.dirname(ns.database)
-+
-+ # Create a downloader
-+ d = location.downloader.Downloader()
-+
-+ # Try downloading a new database
-+ try:
-+ t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
-+
-+ # If no file could be downloaded, log a message
-+ except FileNotFoundError as e:
-+ log.error("Could not download a new database")
-+ return 1
-+
-+ # If we have not received a new file, there is nothing to do
-+ if not t:
-+ return 3
-+
-+ # Move temporary file to destination
-+ shutil.move(t.name, ns.database)
-+
-+ return 0
-+
-+ def handle_verify(self, ns):
-+ try:
-+ db = location.Database(ns.database)
-+ except FileNotFoundError as e:
-+ log.error("%s: %s" % (ns.database, e))
-+ return 127
-+
-+ # Verify the database
-+ with open(ns.public_key, "r") as f:
-+ if not db.verify(f):
-+ log.error("Could not verify database")
-+ return 1
-+
-+ # Success
-+ log.debug("Database successfully verified")
-+ return 0
-+
- def __get_output_formatter(self, ns):
- try:
- cls = self.output_formats[ns.output_format]
-diff --git a/src/python/locationmodule.c b/src/python/locationmodule.c
-index a04cab7..5b72be9 100644
---- a/src/python/locationmodule.c
-+++ b/src/python/locationmodule.c
-@@ -50,9 +50,9 @@ static PyObject* set_log_level(PyObject* m, PyObject* args) {
- }
-
- static PyObject* discover_latest_version(PyObject* m, PyObject* args) {
-- unsigned int version = 0;
-+ unsigned int version = LOC_DATABASE_VERSION_LATEST;
-
-- if (!PyArg_ParseTuple(args, "i", &version))
-+ if (!PyArg_ParseTuple(args, "|i", &version))
- return NULL;
-
- time_t t = 0;
+++ /dev/null
-commit 6bfde1447d237d2a345b99677c5b74e54cbd5739
-Author: Michael Tremer <michael.tremer@ipfire.org>
-Date: Thu Jun 4 10:37:50 2020 +0000
-
- Makefile: Remove accidentially committed hacks for Debian
-
- Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
-
-diff --git a/Makefile.am b/Makefile.am
-index ef57551..c75839c 100644
---- a/Makefile.am
-+++ b/Makefile.am
-@@ -37,9 +37,6 @@ LIBLOC_CURRENT=0
- LIBLOC_REVISION=0
- LIBLOC_AGE=0
-
--pythondir = $(prefix)/lib/python3/dist-packages
--pyexecdir = $(prefix)/lib/python$(PYTHON_VERSION)/lib-dynload
--
- DISTCHECK_CONFIGURE_FLAGS = \
- --with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir)
-
+++ /dev/null
-commit 9df8db2ae6268b0901961625fd27b4dde1ed451f
-Author: Michael Tremer <michael.tremer@ipfire.org>
-Date: Mon Jun 1 18:23:50 2020 +0000
-
- Makefile: Remove Python path overrides for Debian
-
- Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
-
-diff --git a/Makefile.am b/Makefile.am
-index 570ec3a..31869e0 100644
---- a/Makefile.am
-+++ b/Makefile.am
-@@ -54,10 +54,6 @@ SED_PROCESS = \
- databasedir = $(localstatedir)/lib/location
- pkgconfigdir = $(libdir)/pkgconfig
-
--# XXX hardcode path for Debian
--pythondir = $(prefix)/lib/python3/dist-packages
--pyexecdir = $(prefix)/lib/python$(PYTHON_VERSION)/lib-dynload
--
- # Overwrite Python path
- pkgpythondir = $(pythondir)/location
-
-diff --git a/debian/libloc.install b/debian/libloc.install
-index eb3b49a..30bbeca 100644
---- a/debian/libloc.install
-+++ b/debian/libloc.install
-@@ -2,8 +2,7 @@ usr/bin/location-downloader
- usr/bin/location-exporter
- usr/bin/location-query
- usr/lib/*/libloc.so.*
--usr/lib/python3*/dist-packages
--usr/lib/python3*/lib-dynload/*.so
-+usr/lib/python3*/site-packages
- src/systemd/*.service /lib/systemd/system/
- src/systemd/*.timer /lib/systemd/system/
- var/lib/location/signing-key.pem
+++ /dev/null
-commit 1d237439676e8b9ee10a6dde2c64f5ba3a057210
-Author: Michael Tremer <michael.tremer@ipfire.org>
-Date: Wed Jun 3 17:21:31 2020 +0000
-
- Rename location-query(8) to location(8)
-
- Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
-
-diff --git a/Makefile.am b/Makefile.am
-index bf204d8..59870b1 100644
---- a/Makefile.am
-+++ b/Makefile.am
-@@ -238,19 +238,19 @@ uninstall-perl:
- $(DESTDIR)/$(prefix)/man/man3/Location.3pm
-
- bin_SCRIPTS = \
-+ src/python/location \
- src/python/location-exporter \
-- src/python/location-importer \
-- src/python/location-query
-+ src/python/location-importer
-
- EXTRA_DIST += \
-+ src/python/location.in \
- src/python/location-exporter.in \
-- src/python/location-importer.in \
-- src/python/location-query.in
-+ src/python/location-importer.in
-
- CLEANFILES += \
-+ src/python/location \
- src/python/location-exporter \
-- src/python/location-importer \
-- src/python/location-query
-+ src/python/location-importer
-
- # ------------------------------------------------------------------------------
-
-@@ -374,7 +374,7 @@ src_test_signature_LDADD = \
- # ------------------------------------------------------------------------------
-
- MANPAGES = \
-- man/location-query.8
-+ man/location.8
-
- MANPAGES_TXT = $(patsubst %.8,%.txt,$(MANPAGES))
- MANPAGES_HTML = $(patsubst %.txt,%.html,$(MANPAGES_TXT))
-diff --git a/man/location-query.txt b/man/location.txt
-similarity index 84%
-rename from man/location-query.txt
-rename to man/location.txt
-index acb43cd..672c2b2 100644
---- a/man/location-query.txt
-+++ b/man/location.txt
-@@ -1,21 +1,21 @@
--= location-query(8)
-+= location(8)
-
- == NAME
--location-query - Query the location database
-+location - Query the location database
-
- == SYNOPSIS
- [verse]
--`location-query lookup ADDRESS [ADDRESS...]`
--`location-query get-as ASN [ASN...]`
--`location-query search-as STRING`
--`location-query update`
--`location-query verify`
--`location-query list-networks-by-as ASN`
--`location-query list-networks-by-cc COUNTRY_CODE`
--`location-query list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]`
-+`location lookup ADDRESS [ADDRESS...]`
-+`location get-as ASN [ASN...]`
-+`location search-as STRING`
-+`location update`
-+`location verify`
-+`location list-networks-by-as ASN`
-+`location list-networks-by-cc COUNTRY_CODE`
-+`location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]`
-
- == DESCRIPTION
--The `location-query` retrieves information from the location database.
-+`location` retrieves information from the location database.
- This data can be used to determine someone's location on the Internet
- and for building firewall rulesets to block access from certain ASes
- or countries.
-@@ -93,7 +93,7 @@ or countries.
- Shows the program's version and exists.
-
- == EXIT CODES
--The 'location-query' command will normally exit with code zero.
-+The 'location' command will normally exit with code zero.
- If there has been a problem and the requested action could not be performed,
- the exit code is unequal to zero.
-
-diff --git a/src/python/location-query.in b/src/python/location.in
-similarity index 99%
-rename from src/python/location-query.in
-rename to src/python/location.in
-index 0291786..10618e2 100644
---- a/src/python/location-query.in
-+++ b/src/python/location.in
-@@ -248,7 +248,7 @@ class CLI(object):
- try:
- db = location.Database(args.database)
- except FileNotFoundError as e:
-- sys.stderr.write("location-query: Could not open database %s: %s\n" \
-+ sys.stderr.write("location: Could not open database %s: %s\n" \
- % (args.database, e))
- sys.exit(1)
projects / people / pmueller / ipfire-2.x.git / commitdiff
