systemd: Add Documentation= to location-update master
authorValters Jansons <valter.jansons@gmail.com>
Fri, 16 Apr 2021 13:06:12 +0000 (16:06 +0300)
committerMichael Tremer <michael.tremer@ipfire.org>
Mon, 19 Apr 2021 13:15:53 +0000 (13:15 +0000)
Systemd units are expected to provide some documentation information
such as manpages, or direct links, which provide more details about
that unit. This commit simply links location-update.service to the
manual for location(8) followed by a fallback to the online manual.

See: https://manpages.debian.org/buster/systemd/systemd.unit.5.en.html#%5BUNIT%5D_SECTION_OPTIONS

Resolves: lintian: systemd-service-file-missing-documentation-key

Signed-off-by: Valters Jansons <valter.jansons@gmail.com>
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
13 files changed:
Makefile.am
configure.ac
debian/changelog
debian/control
debian/copyright
debian/location-python.examples [new file with mode: 0644]
debian/rules
debian/watch [new file with mode: 0644]
po/POTFILES.in
po/de.po
src/database.c
src/python/location-importer.in
src/systemd/location-update.service.in

index 4b865f1..dbdfd8e 100644 (file)
@@ -79,8 +79,9 @@ update-po: po/POTFILES.in
        $(MAKE) -C po update-po
 
 po/POTFILES.in: Makefile
-       find $(abs_srcdir)/src | \
-               grep -E "\.(in|py)$$" | sed -e "s@$(abs_srcdir)/@@g" | sort > $@
+       find $(abs_srcdir)/src -type f \( -name '*.in' -o -name '*.py' \) \
+               \! -exec git check-ignore -q {} \; -print | \
+               sed -e "s@$(abs_srcdir)/@@g" | LC_ALL=C sort > $@
 
 EXTRA_DIST += \
        examples/private-key.pem \
@@ -228,7 +229,7 @@ build-perl:
        @test -e $(builddir)/src/perl/t/Location.t || ln -s --relative $(srcdir)/src/perl/t/Location.t $(builddir)/src/perl/t/
        @test -e $(builddir)/src/perl/typemap || ln -s --relative $(srcdir)/src/perl/typemap $(builddir)/src/perl/
 
-       cd $(builddir)/src/perl && $(PERL) Makefile.PL INSTALLDIRS=vendor \
+       cd $(builddir)/src/perl && $(PERL) Makefile.PL PREFIX="$(prefix)" \
                INC="-I$(abs_srcdir)/src" LIBS="-L$(abs_builddir)/src/.libs -lloc"
        cd $(builddir)/src/perl && $(MAKE) LD_RUN_PATH=
 
@@ -239,7 +240,7 @@ check-perl: testdata.db
 
 .PHONY: install-perl
 install-perl:
-       cd $(builddir)/src/perl && $(MAKE) install
+       cd $(builddir)/src/perl && $(MAKE) install DESTIDR=$(DESTDIR)
 
 .PHONY: clean-perl
 clean-perl:
index 9eb9012..cc9de27 100644 (file)
@@ -1,6 +1,6 @@
 AC_PREREQ(2.60)
 AC_INIT([libloc],
-        [0.9.5],
+        [0.9.6],
         [location@lists.ipfire.org],
         [libloc],
         [https://location.ipfire.org/])
index 395c292..e58c0ca 100644 (file)
@@ -1,5 +1,18 @@
-libloc (0.9.5-1) UNRELEASED; urgency=medium
+libloc (0.9.6-1) unstable; urgency=medium
 
-  * Initial release. (Closes: #XXXXXX)
+  * location-importer.in: skip networks with unknown country codes
+  * location-importer.in: process unaligned IP ranges in RIR data files
+    correctly
+  * database: Free mmapped countries section
+  * location-importer.in: reduce log noise for unusable networks
+  * location-importer.in: delete 6to4 IPv6 space as well
+  * location-importer.in: fix typo
+  * location: Fix list-networks-by-as
+
+ -- Michael Tremer <michael.tremer@ipfire.org>  Wed, 31 Mar 2021 14:06:00 +0100
+
+libloc (0.9.5-1) unstable; urgency=medium
+
+  * Initial release.
 
  -- Stefan Schantl <stefan.schantl@ipfire.org>  Sun, 27 Oct 2019 18:55:44 +0100
index dc40927..4b1407a 100644 (file)
@@ -16,6 +16,7 @@ Build-Depends:
  systemd,
  xsltproc <!nodoc>,
  docbook-xsl <!nodoc>,
+ git,
 Rules-Requires-Root: no
 Homepage: https://location.ipfire.org/
 Vcs-Git: https://git.ipfire.org/pub/git/location/libloc.git
@@ -23,6 +24,7 @@ Vcs-Browser: https://git.ipfire.org/pub/git/location/libloc.git
 
 Package: libloc1
 Architecture: any
+Section: libs
 Pre-Depends:
  ${misc:Pre-Depends}
 Depends:
@@ -67,7 +69,7 @@ Depends:
  location-python (= ${binary:Version}),
  ${misc:Depends},
  ${python3:Depends}
-Multi-Arch: same
+Multi-Arch: foreign
 Description: Tools to author location databases
  This package contains tools that are required to build location databases
 
@@ -80,6 +82,6 @@ Depends:
  ${misc:Depends},
  ${python3:Depends},
  ${shlibs:Depends}
-Multi-Arch: same
+Multi-Arch: foreign
 Description: Python modules for libloc
  This package contains Python bindings for libloc
index 151d53a..3bd7654 100644 (file)
@@ -1,4 +1,4 @@
-Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Upstream-Name: libloc
 Upstream-Contact: Michael Tremer <michael.tremer@ipfire.org>
 Source: https://location.ipfire.org/download
@@ -8,7 +8,7 @@ Copyright: 2017-2019 IPFire Development team <info@ipfire.org>
 License: LGPL-2.1
 
 Files: debian/*
-Copyright: 2019 Stefan Schantl stefan.schantl@ipfire.org>
+Copyright: 2019 Stefan Schantl <stefan.schantl@ipfire.org>
 License: LGPL-2.1
 
 License: LGPL-2.1
diff --git a/debian/location-python.examples b/debian/location-python.examples
new file mode 100644 (file)
index 0000000..cf2a6ee
--- /dev/null
@@ -0,0 +1 @@
+examples/python/
index 4e544d7..8893b7b 100755 (executable)
@@ -10,6 +10,7 @@ export DEB_BUILD_MAINT_OPTIONS=hardening=+all
        dh $@ --with python3 --with-systemd
 
 override_dh_auto_configure:
+       intltoolize --force --automake
        dh_auto_configure -- --disable-perl
 
 override_dh_perl:
diff --git a/debian/watch b/debian/watch
new file mode 100644 (file)
index 0000000..19ace6d
--- /dev/null
@@ -0,0 +1,3 @@
+version=4
+https://source.ipfire.org/releases/libloc/ \
+    @PACKAGE@@ANY_VERSION@@ARCHIVE_EXT@ debian uupdate
index 9088986..5d2cc46 100644 (file)
@@ -1,11 +1,10 @@
 src/libloc.pc.in
+src/python/__init__.py.in
 src/python/database.py
 src/python/downloader.py
 src/python/export.py
 src/python/i18n.py
 src/python/importer.py
-src/python/__init__.py
-src/python/__init__.py.in
 src/python/location-importer.in
 src/python/location.in
 src/python/logger.py
index c35f348..3b073d6 100644 (file)
--- a/po/de.po
+++ b/po/de.po
@@ -7,7 +7,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: libloc 0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2020-08-04 15:44+0000\n"
+"POT-Creation-Date: 2021-04-15 11:29+0000\n"
 "PO-Revision-Date: 2018-02-01 14:05+0000\n"
 "Last-Translator: Michael Tremer <michael.tremer@ipfire.org>\n"
 "Language-Team: German\n"
@@ -217,8 +217,7 @@ msgstr ""
 msgid "AS%(asn)s belongs to %(name)s"
 msgstr ""
 
-#, python-format
-msgid "The datase has recently be updated recently (%s)"
+msgid "The database has been updated recently"
 msgstr ""
 
 msgid "You must at least pass one flag"
index 4b8bf1d..b92a803 100644 (file)
@@ -446,6 +446,13 @@ static void loc_database_free(struct loc_database* db) {
                        ERROR(db->ctx, "Could not unmap network nodes section: %s\n", strerror(errno));
        }
 
+       // Remove mapped countries section
+       if (db->countries_v1) {
+               r = munmap(db->countries_v1, db->countries_count * sizeof(*db->countries_v1));
+               if (r)
+                       ERROR(db->ctx, "Could not unmap countries section: %s\n", strerror(errno));
+       }
+
        if (db->pool)
                loc_stringpool_unref(db->pool);
 
index 2506925..1e08458 100644 (file)
@@ -3,7 +3,7 @@
 #                                                                             #
 # libloc - A library to determine the location of someone on the Internet     #
 #                                                                             #
-# Copyright (C) 2020 IPFire Development Team <info@ipfire.org>                #
+# Copyright (C) 2020-2021 IPFire Development Team <info@ipfire.org>           #
 #                                                                             #
 # This library is free software; you can redistribute it and/or               #
 # modify it under the terms of the GNU Lesser General Public                  #
@@ -388,10 +388,17 @@ class CLI(object):
                                TRUNCATE TABLE networks;
                        """)
 
+                       # Fetch all valid country codes to check parsed networks aganist...
+                       rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
+                       validcountries = []
+
+                       for row in rows:
+                               validcountries.append(row.country_code)
+
                        for source in location.importer.WHOIS_SOURCES:
                                with downloader.request(source, return_blocks=True) as f:
                                        for block in f:
-                                               self._parse_block(block)
+                                               self._parse_block(block, validcountries)
 
                        # Process all parsed networks from every RIR we happen to have access to,
                        # insert the largest network chunks into the networks table immediately...
@@ -467,7 +474,7 @@ class CLI(object):
                                # Download data
                                with downloader.request(source) as f:
                                        for line in f:
-                                               self._parse_line(line)
+                                               self._parse_line(line, validcountries)
 
        def _check_parsed_network(self, network):
                """
@@ -532,7 +539,7 @@ class CLI(object):
                # be suitable for libloc consumption...
                return True
 
-       def _parse_block(self, block):
+       def _parse_block(self, block, validcountries = None):
                # Get first line to find out what type of block this is
                line = block[0]
 
@@ -542,7 +549,7 @@ class CLI(object):
 
                # inetnum
                if line.startswith("inet6num:") or line.startswith("inetnum:"):
-                       return self._parse_inetnum_block(block)
+                       return self._parse_inetnum_block(block, validcountries)
 
                # organisation
                elif line.startswith("organisation:"):
@@ -573,7 +580,7 @@ class CLI(object):
                        autnum.get("asn"), autnum.get("org"),
                )
 
-       def _parse_inetnum_block(self, block):
+       def _parse_inetnum_block(self, block, validcountries = None):
                log.debug("Parsing inetnum block:")
 
                inetnum = {}
@@ -604,18 +611,10 @@ class CLI(object):
                                        log.warning("Could not parse line: %s" % line)
                                        return
 
-                               # Set prefix to default
-                               prefix = 32
-
-                               # Count number of addresses in this subnet
-                               num_addresses = int(end_address) - int(start_address)
-                               if num_addresses:
-                                       prefix -= math.log(num_addresses, 2)
-
-                               inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
+                               inetnum["inetnum"] = list(ipaddress.summarize_address_range(start_address, end_address))
 
                        elif key == "inet6num":
-                               inetnum[key] = val
+                               inetnum[key] = [ipaddress.ip_network(val, strict=False)]
 
                        elif key == "country":
                                inetnum[key] = val.upper()
@@ -624,21 +623,20 @@ class CLI(object):
                if not inetnum or not "country" in inetnum:
                        return
 
-               # Skip objects with bogus country code 'ZZ'
-               if inetnum.get("country") == "ZZ":
-                       log.warning("Skipping network with bogus country 'ZZ': %s" % \
-                               (inetnum.get("inet6num") or inetnum.get("inetnum")))
+               # Skip objects with unknown country codes
+               if validcountries and inetnum.get("country") not in validcountries:
+                       log.warning("Skipping network with bogus country '%s': %s" % \
+                               (inetnum.get("country"), inetnum.get("inet6num") or inetnum.get("inetnum")))
                        return
 
-               network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
-
-               if not self._check_parsed_network(network):
-                       return
-
-               self.db.execute("INSERT INTO _rirdata(network, country) \
-                       VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
-                       "%s" % network, inetnum.get("country"),
-               )
+               # Iterate through all networks enumerated from above, check them for plausibility and insert
+               # them into the database, if _check_parsed_network() succeeded
+               for single_network in inetnum.get("inet6num") or inetnum.get("inetnum"):
+                       if self._check_parsed_network(single_network):
+                               self.db.execute("INSERT INTO _rirdata(network, country) \
+                                       VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
+                                       "%s" % single_network, inetnum.get("country"),
+                               )
 
        def _parse_org_block(self, block):
                org = {}
@@ -661,7 +659,7 @@ class CLI(object):
                        org.get("organisation"), org.get("org-name"),
                )
 
-       def _parse_line(self, line):
+       def _parse_line(self, line, validcountries = None):
                # Skip version line
                if line.startswith("2"):
                        return
@@ -676,8 +674,15 @@ class CLI(object):
                        log.warning("Could not parse line: %s" % line)
                        return
 
-               # Skip any lines that are for stats only
-               if country_code == "*":
+               # Skip any lines that are for stats only or do not have a country
+               # code at all (avoids log spam below)
+               if not country_code or country_code == '*':
+                       return
+
+               # Skip objects with unknown country codes
+               if validcountries and country_code not in validcountries:
+                       log.warning("Skipping line with bogus country '%s': %s" % \
+                               (country_code, line))
                        return
 
                if type in ("ipv6", "ipv4"):
index daae2c5..1c8e116 100644 (file)
@@ -1,5 +1,6 @@
 [Unit]
 Description=Automatic Location Database Updater
+Documentation=man:location(8) https://man-pages.ipfire.org/libloc/location.html
 Requires=network.target
 
 [Service]