From: Jochen Sprickerhof Date: Sat, 29 Oct 2022 16:09:19 +0000 (+0200) Subject: New upstream version 0.9.16 X-Git-Tag: upstream/0.9.16^0 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=b1863b6451e1e020cbe1096d8a5623507616c562;p=location%2Fdebian%2Flibloc.git New upstream version 0.9.16 --- diff --git a/Makefile.am b/Makefile.am index 81bd992..7f0d8d0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -56,8 +56,7 @@ databasedir = $(localstatedir)/lib/location pkgconfigdir = $(libdir)/pkgconfig # Overwrite Python path -#pkgpythondir = $(pythondir)/location -pkgpythondir = /usr/lib/python3/dist-packages/location +pkgpythondir = $(pythondir)/location %: %.in Makefile $(SED_PROCESS) @@ -233,8 +232,7 @@ EXTRA_DIST += \ src/perl/t/Location.t \ src/perl/typemap -.PHONY: build-perl -build-perl: +build-perl: src/libloc.la @mkdir -p $(builddir)/src/perl/{lib,t} @test -e $(builddir)/src/perl/Location.xs || ln -s --relative $(srcdir)/src/perl/Location.xs $(builddir)/src/perl/ @test -e $(builddir)/src/perl/MANIFEST || ln -s --relative $(srcdir)/src/perl/MANIFEST $(builddir)/src/perl/ @@ -243,30 +241,33 @@ build-perl: @test -e $(builddir)/src/perl/t/Location.t || ln -s --relative $(srcdir)/src/perl/t/Location.t $(builddir)/src/perl/t/ @test -e $(builddir)/src/perl/typemap || ln -s --relative $(srcdir)/src/perl/typemap $(builddir)/src/perl/ - cd $(builddir)/src/perl && $(PERL) Makefile.PL PREFIX="$(prefix)" \ + cd $(builddir)/src/perl && $(PERL) Makefile.PL NO_PACKLIST=1 NO_PERLLOCAL=1 \ + INSTALLDIRS=vendor \ INC="-I$(abs_srcdir)/src" LIBS="-L$(abs_builddir)/src/.libs -lloc" - cd $(builddir)/src/perl && $(MAKE) LD_RUN_PATH= + cd $(builddir)/src/perl && $(MAKE) + touch build-perl .PHONY: check-perl -check-perl: testdata.db +check-perl: testdata.db build-perl cd $(builddir)/src/perl && $(MAKE) LD_LIBRARY_PATH="$(abs_builddir)/src/.libs" test \ database="../../$<" keyfile="$(abs_srcdir)/examples/public-key.pem" .PHONY: install-perl -install-perl: - cd $(builddir)/src/perl && $(MAKE) install DESTIDR=$(DESTDIR) +install-perl: build-perl + cd $(builddir)/src/perl && $(MAKE) install DESTDIR=$(DESTDIR) .PHONY: clean-perl clean-perl: cd $(builddir)/src/perl && $(MAKE) distclean + rm -f build-perl .PHONY: uninstall-perl uninstall-perl: - rm -rvf \ - $(DESTDIR)/$(prefix)/lib/*/perl/*/Location.pm \ - $(DESTDIR)/$(prefix)/lib/*/perl/*/auto/Location \ - $(DESTDIR)/$(prefix)/lib/*/perl/*/perllocal.pod \ - $(DESTDIR)/$(prefix)/man/man3/Location.3pm + rm -vf \ + $(DESTDIR)/@PERL_MODPATH@/Location.pm \ + $(DESTDIR)/@PERL_MODPATH@/auto/Location/Location.so \ + $(DESTDIR)/@PERL_MANPATH@/Location.3pm + -rmdir $(DESTDIR)/@PERL_MODPATH@/auto/Location bin_SCRIPTS = \ src/scripts/location \ @@ -311,7 +312,15 @@ EXTRA_DIST += \ # ------------------------------------------------------------------------------ dist_database_DATA = \ - src/signing-key.pem + data/database.db \ + data/signing-key.pem + +install-data-hook: + chmod 444 $(DESTDIR)$(databasedir)/database.db + +.PHONY: update-database +update-database: + curl https://location.ipfire.org/databases/1/location.db.xz | xz -d > data/database.db # ------------------------------------------------------------------------------ @@ -326,16 +335,12 @@ TESTS_LDADD = \ TESTS_ENVIRONMENT = \ PYTHONPATH=$(abs_srcdir)/src/python:$(abs_builddir)/src/python/.libs \ - TEST_DATA_DIR="$(abs_top_srcdir)/tests/data" + TEST_DATA_DIR="$(abs_top_srcdir)/data" TESTS = \ $(check_PROGRAMS) \ $(dist_check_SCRIPTS) -EXTRA_DIST += \ - tests/data/location-2022-03-30.db \ - tests/data/signing-key.pem - CLEANFILES += \ testdata.db @@ -444,7 +449,7 @@ src_test_address_LDADD = \ MANPAGES = \ $(MANPAGES_3) \ - $(MANPAGES_8) + $(MANPAGES_1) MANPAGES_3 = \ man/libloc.3 \ @@ -458,12 +463,12 @@ MANPAGES_3 = \ man/loc_set_log_fn.3 \ man/loc_set_log_priority.3 -MANPAGES_8 = \ - man/location.8 +MANPAGES_1 = \ + man/location.1 -MANPAGES_TXT = $(MANPAGES_TXT_3) $(MANPAGES_TXT_8) +MANPAGES_TXT = $(MANPAGES_TXT_3) $(MANPAGES_TXT_1) MANPAGES_TXT_3 = $(patsubst %.3,%.txt,$(MANPAGES_3)) -MANPAGES_TXT_8 = $(patsubst %.8,%.txt,$(MANPAGES_8)) +MANPAGES_TXT_1 = $(patsubst %.1,%.txt,$(MANPAGES_1)) MANPAGES_HTML = $(patsubst %.txt,%.html,$(MANPAGES_TXT)) MANPAGES_XML = $(patsubst %.txt,%.xml,$(MANPAGES_TXT)) @@ -511,7 +516,7 @@ man/%.xml: man/%.txt man/asciidoc.conf man/%.3: man/%.xml $(XSLTPROC_COMMAND_MAN) -man/%.8: man/%.xml +man/%.1: man/%.xml $(XSLTPROC_COMMAND_MAN) man/%.html: man/%.txt man/asciidoc.conf diff --git a/configure.ac b/configure.ac index 9fad1b9..96e6b0e 100644 --- a/configure.ac +++ b/configure.ac @@ -1,6 +1,6 @@ AC_PREREQ(2.60) AC_INIT([libloc], - [0.9.15], + [0.9.16], [location@lists.ipfire.org], [libloc], [https://location.ipfire.org/]) @@ -122,7 +122,7 @@ CC_CHECK_FLAGS_APPEND([my_LDFLAGS], [LDFLAGS], [-fno-semantic-interposition]) AC_ARG_WITH([database-path], AS_HELP_STRING([--with-database-path], [The default database path]), - [], [with_database_path=/var/lib/${PACKAGE_NAME}/database.db] + [], [with_database_path=/var/lib/location/database.db] ) if test -z "${with_database_path}"; then @@ -175,10 +175,18 @@ PKG_CHECK_MODULES([PYTHON], [python-${PYTHON_VERSION}]) AC_PATH_PROG(PERL, perl, no) AC_SUBST(PERL) -AX_PROG_PERL_MODULES(ExtUtils::MakeMaker,, AC_MSG_WARN(Need some Perl modules)) +AX_PROG_PERL_MODULES(Config ExtUtils::MakeMaker,, AC_MSG_WARN(Need some Perl modules)) AC_ARG_ENABLE(perl, AS_HELP_STRING([--disable-perl], [do not build the perl modules]), [],[enable_perl=yes]) AM_CONDITIONAL(ENABLE_PERL, test "$enable_perl" = "yes") +AS_IF([test "$enable_perl" = "yes"], + [ + PERL_MODPATH=$($PERL -MConfig -e 'print $Config{installvendorarch}') + PERL_MANPATH=$($PERL -MConfig -e 'print $Config{installvendorman3dir}') + AC_SUBST(PERL_MODPATH) + AC_SUBST(PERL_MANPATH) + ], +) dnl Checking for libresolv case "${host}" in @@ -223,5 +231,7 @@ AC_MSG_RESULT([ systemd support: ${have_systemd} Bindings: - perl: ${enable_perl} + Perl: ${enable_perl} + Perl module path: ${PERL_MODPATH} + Perl manual path: ${PERL_MANPATH} ]) diff --git a/tests/data/location-2022-03-30.db b/data/database.db similarity index 80% rename from tests/data/location-2022-03-30.db rename to data/database.db index fff8d34..b36cc8a 100644 Binary files a/tests/data/location-2022-03-30.db and b/data/database.db differ diff --git a/src/signing-key.pem b/data/signing-key.pem similarity index 100% rename from src/signing-key.pem rename to data/signing-key.pem diff --git a/man/libloc.txt b/man/libloc.txt index baf98c1..ec14e16 100644 --- a/man/libloc.txt +++ b/man/libloc.txt @@ -39,7 +39,7 @@ either version 2.1 of the License, or (at your option) any later version. == See Also -link:location[8] +link:location[1] == Bug Reports diff --git a/man/location.txt b/man/location.txt index 3dfddf5..70352d2 100644 --- a/man/location.txt +++ b/man/location.txt @@ -1,4 +1,4 @@ -= location(8) += location(1) == NAME location - Query the location database diff --git a/src/database.c b/src/database.c index be60aa4..617b61e 100644 --- a/src/database.c +++ b/src/database.c @@ -641,8 +641,11 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) { } } + int sig1_valid = 0; + int sig2_valid = 0; + // Check first signature - if (db->signature1.data) { + if (db->signature1.length) { hexdump(db->ctx, db->signature1.data, db->signature1.length); r = EVP_DigestVerifyFinal(mdctx, @@ -650,19 +653,19 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) { if (r == 0) { DEBUG(db->ctx, "The first signature is invalid\n"); - r = 1; } else if (r == 1) { DEBUG(db->ctx, "The first signature is valid\n"); - r = 0; + sig1_valid = 1; } else { ERROR(db->ctx, "Error verifying the first signature: %s\n", ERR_error_string(ERR_get_error(), NULL)); r = -1; + goto CLEANUP; } } // Check second signature only when the first one was invalid - if (r && db->signature2.data) { + if (db->signature2.length) { hexdump(db->ctx, db->signature2.data, db->signature2.length); r = EVP_DigestVerifyFinal(mdctx, @@ -670,14 +673,14 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) { if (r == 0) { DEBUG(db->ctx, "The second signature is invalid\n"); - r = 1; } else if (r == 1) { DEBUG(db->ctx, "The second signature is valid\n"); - r = 0; + sig2_valid = 1; } else { ERROR(db->ctx, "Error verifying the second signature: %s\n", ERR_error_string(ERR_get_error(), NULL)); r = -1; + goto CLEANUP; } } @@ -685,6 +688,12 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) { INFO(db->ctx, "Signature checked in %.4fms\n", (double)(end - start) / CLOCKS_PER_SEC * 1000); + // Check if at least one signature as okay + if (sig1_valid || sig2_valid) + r = 0; + else + r = 1; + CLEANUP: // Cleanup EVP_MD_CTX_free(mdctx); diff --git a/src/python/location/__init__.py b/src/python/location/__init__.py index f635737..e0ba510 100644 --- a/src/python/location/__init__.py +++ b/src/python/location/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/python3 ############################################################################### # # # libloc - A library to determine the location of someone on the Internet # @@ -19,6 +18,7 @@ # Import everything from the C module from _location import * +from _location import __version__ # Initialise logging from . import logger diff --git a/src/python/location/database.py b/src/python/location/database.py index 5d79941..b97d93a 100644 --- a/src/python/location/database.py +++ b/src/python/location/database.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - """ A lightweight wrapper around psycopg2. diff --git a/src/python/location/downloader.py b/src/python/location/downloader.py index b9e0c22..3618968 100644 --- a/src/python/location/downloader.py +++ b/src/python/location/downloader.py @@ -1,4 +1,3 @@ -#!/usr/bin/python3 ############################################################################### # # # libloc - A library to determine the location of someone on the Internet # diff --git a/src/python/location/export.py b/src/python/location/export.py index f5ed37f..25a532e 100644 --- a/src/python/location/export.py +++ b/src/python/location/export.py @@ -1,4 +1,3 @@ -#!/usr/bin/python3 ############################################################################### # # # libloc - A library to determine the location of someone on the Internet # @@ -48,6 +47,9 @@ class OutputWriter(object): self.family = family self.directory = directory + # Tag + self.tag = self._make_tag() + # Open output file if f: self.f = f @@ -58,9 +60,6 @@ class OutputWriter(object): else: self.f = io.StringIO() - # Tag - self.tag = self._make_tag() - # Call any custom initialization self.init() diff --git a/src/python/location/i18n.py b/src/python/location/i18n.py index 2161aa6..c97c51c 100644 --- a/src/python/location/i18n.py +++ b/src/python/location/i18n.py @@ -1,4 +1,3 @@ -#!/usr/bin/python3 ############################################################################### # # # libloc - A library to determine the location of someone on the Internet # diff --git a/src/python/location/importer.py b/src/python/location/importer.py index 96f2218..d285162 100644 --- a/src/python/location/importer.py +++ b/src/python/location/importer.py @@ -1,4 +1,3 @@ -#!/usr/bin/python3 ############################################################################### # # # libloc - A library to determine the location of someone on the Internet # diff --git a/src/python/location/logger.py b/src/python/location/logger.py index 0bdf9ec..62ad8fb 100644 --- a/src/python/location/logger.py +++ b/src/python/location/logger.py @@ -1,4 +1,3 @@ -#!/usr/bin/python3 ############################################################################### # # # libloc - A library to determine the location of someone on the Internet # diff --git a/src/python/writer.c b/src/python/writer.c index 5d8027c..1c06384 100644 --- a/src/python/writer.c +++ b/src/python/writer.c @@ -67,8 +67,8 @@ static int Writer_init(WriterObject* self, PyObject* args, PyObject* kwargs) { return -1; // Re-open file descriptor - f2 = fdopen(fd, "r"); - if (!f2) { + f1 = fdopen(fd, "r"); + if (!f1) { PyErr_SetFromErrno(PyExc_IOError); return -1; } diff --git a/src/scripts/location-importer.in b/src/scripts/location-importer.in index 8d47497..9faf23b 100644 --- a/src/scripts/location-importer.in +++ b/src/scripts/location-importer.in @@ -1341,8 +1341,10 @@ class CLI(object): "ap-southeast-2": "AU", "ap-southeast-3": "MY", "ap-southeast-4": "AU", + "ap-southeast-6": "AP", # XXX: Precise location not documented anywhere "ap-northeast-1": "JP", "ca-central-1": "CA", + "ca-west-1": "CA", "eu-central-1": "DE", "eu-central-2": "CH", "eu-west-1": "IE", @@ -1427,37 +1429,34 @@ class CLI(object): def _update_overrides_for_spamhaus_drop(self): downloader = location.importer.Downloader() - ip_urls = [ - "https://www.spamhaus.org/drop/drop.txt", - "https://www.spamhaus.org/drop/edrop.txt", - "https://www.spamhaus.org/drop/dropv6.txt" + ip_lists = [ + ("SPAMHAUS-DROP", "https://www.spamhaus.org/drop/drop.txt"), + ("SPAMHAUS-EDROP", "https://www.spamhaus.org/drop/edrop.txt"), + ("SPAMHAUS-DROPV6", "https://www.spamhaus.org/drop/dropv6.txt") ] - asn_urls = [ - "https://www.spamhaus.org/drop/asndrop.txt" + asn_lists = [ + ("SPAMHAUS-ASNDROP", "https://www.spamhaus.org/drop/asndrop.txt") ] - for url in ip_urls: - # Fetch IP list + for name, url in ip_lists: + # Fetch IP list from given URL f = downloader.retrieve(url) # Split into lines fcontent = f.readlines() - # Conduct a very basic sanity check to rule out CDN issues causing bogus DROP - # downloads. - if len(fcontent) > 10: - self.db.execute(""" - DELETE FROM autnum_overrides WHERE source = 'Spamhaus ASN-DROP list'; - DELETE FROM network_overrides WHERE source = 'Spamhaus DROP lists'; - """) - else: - log.error("Spamhaus DROP URL %s returned likely bogus file, ignored" % url) - continue - - # Iterate through every line, filter comments and add remaining networks to - # the override table in case they are valid... with self.db.transaction(): + # Conduct a very basic sanity check to rule out CDN issues causing bogus DROP + # downloads. + if len(fcontent) > 10: + self.db.execute("DELETE FROM network_overrides WHERE source = %s", name) + else: + log.error("%s (%s) returned likely bogus file, ignored" % (name, url)) + continue + + # Iterate through every line, filter comments and add remaining networks to + # the override table in case they are valid... for sline in fcontent: # The response is assumed to be encoded in UTF-8... sline = sline.decode("utf-8") @@ -1475,8 +1474,8 @@ class CLI(object): # Sanitize parsed networks... if not self._check_parsed_network(network): - log.warning("Skipping bogus network found in Spamhaus DROP URL %s: %s" % \ - (url, network)) + log.warning("Skipping bogus network found in %s (%s): %s" % \ + (name, url, network)) continue # Conduct SQL statement... @@ -1488,17 +1487,28 @@ class CLI(object): ) VALUES (%s, %s, %s) ON CONFLICT (network) DO UPDATE SET is_drop = True""", "%s" % network, - "Spamhaus DROP lists", + name, True ) - for url in asn_urls: + for name, url in asn_lists: # Fetch URL f = downloader.retrieve(url) - # Iterate through every line, filter comments and add remaining ASNs to - # the override table in case they are valid... + # Split into lines + fcontent = f.readlines() + with self.db.transaction(): + # Conduct a very basic sanity check to rule out CDN issues causing bogus DROP + # downloads. + if len(fcontent) > 10: + self.db.execute("DELETE FROM autnum_overrides WHERE source = %s", name) + else: + log.error("%s (%s) returned likely bogus file, ignored" % (name, url)) + continue + + # Iterate through every line, filter comments and add remaining ASNs to + # the override table in case they are valid... for sline in f.readlines(): # The response is assumed to be encoded in UTF-8... sline = sline.decode("utf-8") @@ -1518,8 +1528,8 @@ class CLI(object): # Filter invalid ASNs... if not self._check_parsed_asn(asn): - log.warning("Skipping bogus ASN found in Spamhaus DROP URL %s: %s" % \ - (url, asn)) + log.warning("Skipping bogus ASN found in %s (%s): %s" % \ + (name, url, asn)) continue # Conduct SQL statement... @@ -1531,7 +1541,7 @@ class CLI(object): ) VALUES (%s, %s, %s) ON CONFLICT (number) DO UPDATE SET is_drop = True""", "%s" % asn, - "Spamhaus ASN-DROP list", + name, True ) diff --git a/src/systemd/location-update.service.in b/src/systemd/location-update.service.in index 1c8e116..50ba01f 100644 --- a/src/systemd/location-update.service.in +++ b/src/systemd/location-update.service.in @@ -1,6 +1,6 @@ [Unit] Description=Automatic Location Database Updater -Documentation=man:location(8) https://man-pages.ipfire.org/libloc/location.html +Documentation=man:location(1) https://man-pages.ipfire.org/libloc/location.html Requires=network.target [Service] diff --git a/src/test-signature.c b/src/test-signature.c index 9af9236..e1be5b1 100644 --- a/src/test-signature.c +++ b/src/test-signature.c @@ -94,7 +94,7 @@ int main(int argc, char** argv) { } // Open another public key - public_key = freopen(ABS_SRCDIR "/src/signing-key.pem", "r", public_key); + public_key = freopen(ABS_SRCDIR "/data/signing-key.pem", "r", public_key); if (!public_key) { fprintf(stderr, "Could not open public key file: %m\n"); exit(EXIT_FAILURE); diff --git a/src/writer.c b/src/writer.c index 51e9a8e..beffcf2 100644 --- a/src/writer.c +++ b/src/writer.c @@ -102,6 +102,13 @@ LOC_EXPORT int loc_writer_new(struct loc_ctx* ctx, struct loc_writer** writer, return r; } + // Add an empty string to the stringpool + r = loc_stringpool_add(w->pool, ""); + if (r) { + loc_writer_unref(w); + return r; + } + // Initialize the network tree r = loc_network_tree_new(ctx, &w->networks); if (r) { @@ -740,7 +747,7 @@ LOC_EXPORT int loc_writer_write(struct loc_writer* writer, FILE* f, enum loc_dat if (writer->signature2_length) { DEBUG(writer->ctx, "Copying second signature of %zu byte(s)\n", - writer->signature1_length); + writer->signature2_length); memcpy(header.signature2, writer->signature2, writer->signature2_length); header.signature2_length = htobe16(writer->signature2_length); diff --git a/tests/data/signing-key.pem b/tests/data/signing-key.pem deleted file mode 120000 index b1da823..0000000 --- a/tests/data/signing-key.pem +++ /dev/null @@ -1 +0,0 @@ -../../src/signing-key.pem \ No newline at end of file diff --git a/tests/python/test-database.py b/tests/python/test-database.py index 4846e7a..1c3448b 100755 --- a/tests/python/test-database.py +++ b/tests/python/test-database.py @@ -25,7 +25,7 @@ TEST_DATA_DIR = os.environ["TEST_DATA_DIR"] class Test(unittest.TestCase): def setUp(self): - path = os.path.join(TEST_DATA_DIR, "location-2022-03-30.db") + path = os.path.join(TEST_DATA_DIR, "database.db") # Load the database self.db = location.Database(path) @@ -45,7 +45,7 @@ class Test(unittest.TestCase): self.assertEqual(self.db.license, "CC BY-SA 4.0") # Created At - self.assertEqual(self.db.created_at, 1648619023) + self.assertIsInstance(self.db.created_at, int) def test_fetch_network(self): """ diff --git a/tests/python/test-export.py b/tests/python/test-export.py index 419b105..6921861 100755 --- a/tests/python/test-export.py +++ b/tests/python/test-export.py @@ -25,7 +25,7 @@ TEST_DATA_DIR = os.environ["TEST_DATA_DIR"] class Test(unittest.TestCase): def setUp(self): - path = os.path.join(TEST_DATA_DIR, "location-2022-03-30.db") + path = os.path.join(TEST_DATA_DIR, "database.db") # Load the database self.db = location.Database(path)