From 7880c1340e2a8cd119c3ed4d82d50a7bd30d8a25 Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Tue, 16 Aug 2022 12:49:49 +0200 Subject: [PATCH] New upstream version 0.9.14 --- .gitignore | 5 +- Makefile.am | 55 +- configure.ac | 4 +- debian/build.sh | 5 +- debian/changelog | 26 + debian/compat | 1 - debian/control | 115 ++-- debian/copyright | 513 +++++++++++++++++- debian/libloc-dev.install | 1 + debian/libloc1.install | 1 + debian/libloc1.symbols | 2 + debian/location-importer.install | 3 - debian/location-perl.install | 1 - debian/location-python.install | 1 - debian/location.install | 6 +- debian/location.manpages | 1 - debian/location.postinst | 14 + debian/location.postrm | 15 + ...hon.examples => python3-location.examples} | 0 debian/python3-location.install | 1 + debian/rules | 35 +- debian/watch | 2 +- po/POTFILES.in | 18 +- src/python/database.c | 6 + .../{__init__.py.in => location/__init__.py} | 2 - src/python/{ => location}/database.py | 0 src/python/{ => location}/downloader.py | 3 +- src/python/{ => location}/export.py | 16 +- src/python/{ => location}/i18n.py | 0 src/python/{ => location}/importer.py | 122 +++-- src/python/{ => location}/logger.py | 0 src/python/locationmodule.c | 4 + src/{python => scripts}/location-importer.in | 380 +++++++------ src/{python => scripts}/location.in | 2 +- 34 files changed, 1001 insertions(+), 359 deletions(-) mode change 100644 => 100755 debian/build.sh delete mode 100644 debian/compat delete mode 100644 debian/location-importer.install delete mode 100644 debian/location-perl.install delete mode 100644 debian/location-python.install delete mode 100644 debian/location.manpages create mode 100644 debian/location.postinst create mode 100644 debian/location.postrm rename debian/{location-python.examples => python3-location.examples} (100%) create mode 100644 debian/python3-location.install rename src/python/{__init__.py.in => location/__init__.py} (98%) rename src/python/{ => location}/database.py (100%) rename src/python/{ => location}/downloader.py (98%) rename src/python/{ => location}/export.py (97%) rename src/python/{ => location}/i18n.py (100%) rename src/python/{ => location}/importer.py (74%) rename src/python/{ => location}/logger.py (100%) rename src/{python => scripts}/location-importer.in (85%) rename src/{python => scripts}/location.in (99%) diff --git a/.gitignore b/.gitignore index f04b70e..20bc895 100644 --- a/.gitignore +++ b/.gitignore @@ -15,9 +15,8 @@ Makefile.in /*.db.xz /libtool /stamp-h1 -/src/python/location -/src/python/location-importer -/src/python/__init__.py +/src/scripts/location +/src/scripts/location-importer /src/systemd/location-update.service /src/systemd/location-update.timer /test.db diff --git a/Makefile.am b/Makefile.am index 983cb4a..eef75c2 100644 --- a/Makefile.am +++ b/Makefile.am @@ -55,7 +55,8 @@ databasedir = $(localstatedir)/lib/location pkgconfigdir = $(libdir)/pkgconfig # Overwrite Python path -pkgpythondir = $(pythondir)/location +#pkgpythondir = $(pythondir)/location +pkgpythondir = /usr/lib/python3/dist-packages/location %: %.in Makefile $(SED_PROCESS) @@ -175,21 +176,13 @@ CLEANFILES += \ src/libloc.pc dist_pkgpython_PYTHON = \ - src/python/database.py \ - src/python/downloader.py \ - src/python/export.py \ - src/python/i18n.py \ - src/python/importer.py \ - src/python/logger.py - -pkgpython_PYTHON = \ - src/python/__init__.py - -EXTRA_DIST += \ - src/python/__init__.py.in - -CLEANFILES += \ - src/python/__init__.py + src/python/location/__init__.py \ + src/python/location/database.py \ + src/python/location/downloader.py \ + src/python/location/export.py \ + src/python/location/i18n.py \ + src/python/location/importer.py \ + src/python/location/logger.py pyexec_LTLIBRARIES = \ src/python/_location.la @@ -275,16 +268,16 @@ uninstall-perl: $(DESTDIR)/$(prefix)/man/man3/Location.3pm bin_SCRIPTS = \ - src/python/location \ - src/python/location-importer + src/scripts/location \ + src/scripts/location-importer EXTRA_DIST += \ - src/python/location.in \ - src/python/location-importer.in + src/scripts/location.in \ + src/scripts/location-importer.in CLEANFILES += \ - src/python/location \ - src/python/location-importer + src/scripts/location \ + src/scripts/location-importer # ------------------------------------------------------------------------------ @@ -321,6 +314,7 @@ TESTS_LDADD = \ src/libloc-internal.la TESTS_ENVIRONMENT = \ + PYTHONPATH=$(abs_srcdir)/src/python:$(abs_builddir)/src/python/.libs \ TEST_DATA_DIR="$(abs_top_srcdir)/tests/data" TESTS = \ @@ -334,7 +328,7 @@ CLEANFILES += \ testdata.db testdata.db: examples/python/create-database.py - PYTHONPATH=$(abs_builddir)/src/python/.libs \ + PYTHONPATH=$(abs_srcdir)/src/python:$(abs_builddir)/src/python/.libs \ ABS_SRCDIR="$(abs_srcdir)" \ $(PYTHON) $< $@ @@ -520,16 +514,21 @@ upload-man: $(MANPAGES_HTML) EXTRA_DIST += \ debian/build.sh \ debian/changelog \ - debian/compat \ debian/control \ debian/copyright \ - debian/location.install \ - debian/location.manpages \ - debian/location-python.install \ + debian/genchangelog.sh \ + debian/gensymbols.sh \ debian/libloc1.install \ + debian/libloc1.symbols \ debian/libloc-dev.install \ + debian/location.install \ + debian/location.postinst \ + debian/location.postrm \ + debian/python3-location.examples \ + debian/python3-location.install \ debian/rules \ - debian/source/format + debian/source/format \ + debian/watch .PHONY: debian debian: dist diff --git a/configure.ac b/configure.ac index e96e9ce..b2db205 100644 --- a/configure.ac +++ b/configure.ac @@ -1,6 +1,6 @@ AC_PREREQ(2.60) AC_INIT([libloc], - [0.9.13], + [0.9.14], [location@lists.ipfire.org], [libloc], [https://location.ipfire.org/]) @@ -154,7 +154,7 @@ AM_CONDITIONAL(HAVE_SYSTEMD, [test "x$have_systemd" = "xyes"]) # ------------------------------------------------------------------------------ # Python -AM_PATH_PYTHON([3.9]) +AM_PATH_PYTHON([3.4]) PKG_CHECK_MODULES([PYTHON], [python-${PYTHON_VERSION}]) # Perl diff --git a/debian/build.sh b/debian/build.sh old mode 100644 new mode 100755 index a11f3a3..a1f6b6c --- a/debian/build.sh +++ b/debian/build.sh @@ -38,6 +38,9 @@ main() { local release for release in ${RELEASES[@]}; do local chroot="${release}-${host_arch}-sbuild" + if [ "${release}" = "buster" ]; then + local buster_backport=( --extra-repository "deb http://deb.debian.org/debian buster-backports main" --build-dep-resolver=aspcud ) + fi mkdir -p "${release}" pushd "${release}" @@ -62,7 +65,7 @@ main() { cp -r "${tmp}/sources" . # Run the build process - if ! sbuild --dist="${release}" --host="${arch}" --source "sources/${package}"; then + if ! sbuild --dist="${release}" --host="${arch}" --source "${buster_backport[@]}" "sources/${package}"; then echo "Could not build package for ${release} on ${arch}" >&2 rm -rf "${tmp}" return 1 diff --git a/debian/changelog b/debian/changelog index 2085fe4..248958e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,29 @@ +libloc (0.9.14-1) unstable; urgency=medium + + [ Michael Tremer ] + * Revert "configure: Require Python >= 3.9" + * Make sources around that we can run tests without location installed + * downloader: Fetch __version__ from C module + * export: Drop using functools + * verify: Show message on success + * export: Don't fail when output stream isn't seekable + * importer: Actually perform the Spamhaus sanity check + * importer: Change download behaviour + * importer: Move importing extended sources/ARIN into transaction + * python: database: Return None if no description/vendor/license set + * importer: Try to make parsing blocks faster + * importer: Import each source individually + * python: Fix missing bracket + * importer: Tolerate that data might exist from other RIRs + * importer: Import all sources in alphabetical order + + [ Peter Müller ] + * location-importer: Only delete override data if we are sure to have + a valid replacement + * location-importer: AS names starting with "DNIC" actually are valid + + -- Michael Tremer Sun, 14 Aug 2022 12:24:16 +0000 + libloc (0.9.13-1) unstable; urgency=medium [ Michael Tremer ] diff --git a/debian/compat b/debian/compat deleted file mode 100644 index f599e28..0000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -10 diff --git a/debian/control b/debian/control index 4b1407a..918c0f6 100644 --- a/debian/control +++ b/debian/control @@ -1,40 +1,40 @@ Source: libloc Maintainer: Stefan Schantl -Section: misc +Section: net Priority: optional -Standards-Version: 4.3.0 +Standards-Version: 4.6.1 Build-Depends: - debhelper (>= 11), - dh-python , - asciidoc , - intltool (>=0.40.0), - libpython3-dev , + debhelper-compat (= 13), + dh-sequence-python3, + asciidoc, + intltool, libssl-dev, libsystemd-dev, - python3-dev:any , pkg-config, + python3-all-dev, systemd, - xsltproc , - docbook-xsl , - git, + xsltproc, + docbook-xsl, Rules-Requires-Root: no Homepage: https://location.ipfire.org/ -Vcs-Git: https://git.ipfire.org/pub/git/location/libloc.git -Vcs-Browser: https://git.ipfire.org/pub/git/location/libloc.git +Vcs-Git: https://salsa.debian.org/debian/libloc.git +Vcs-Browser: https://salsa.debian.org/debian/libloc +Description: IP geolocation query library + libloc is a lightweight library to query the IPFire Location database and + determine the location of someone else on the Internet based on their IP + address. Package: libloc1 Architecture: any Section: libs -Pre-Depends: - ${misc:Pre-Depends} Depends: ${shlibs:Depends}, - ${misc:Depends} -Recommends: - location (= ${binary:Version}) + ${misc:Depends}, Multi-Arch: same -Description: Location library - A library to determine the location of someone on the Internet +Description: ${source:Synopsis} + ${source:Extended-Description} + . + This package provides the shared library. Package: libloc-dev Architecture: any @@ -42,46 +42,59 @@ Section: libdevel Depends: libloc1 (= ${binary:Version}), ${misc:Depends}, -Suggests: - pkg-config Multi-Arch: same -Description: Development files for libloc - Install this package if you wish to develop your own programs using - libloc. +Description: ${source:Synopsis} (development files) + ${source:Extended-Description} + . + This package provides the headers and development files needed to use libloc + in your own programs. Package: location -Architecture: any -Pre-Depends: - ${misc:Pre-Depends} -Depends: - location-python (= ${binary:Version}), - ${misc:Depends}, - ${python3:Depends} -Multi-Arch: same -Description: CLI utilities for libloc - Commands to determine someone's location on the Internet - -Package: location-importer -Architecture: any -Pre-Depends: - ${misc:Pre-Depends} +Architecture: all Depends: - location-python (= ${binary:Version}), + python3-location, ${misc:Depends}, - ${python3:Depends} -Multi-Arch: foreign -Description: Tools to author location databases - This package contains tools that are required to build location databases + ${python3:Depends}, +Recommends: + libloc-database, +Replaces: location-importer (<< 0.9.14-1~) +Breaks: location-importer (<< 0.9.14-1~) +Description: ${source:Synopsis} (CLI utilities) + ${source:Extended-Description} + . + This package provides CLI utilities based on libloc. -Package: location-python +Package: python3-location Architecture: any Section: python -Pre-Depends: - ${misc:Pre-Depends} Depends: ${misc:Depends}, ${python3:Depends}, - ${shlibs:Depends} + ${shlibs:Depends}, + python3-psycopg2, +Replaces: + location-python (<< 0.9.14-1~), +Breaks: + location-python (<< 0.9.14-1~), + location-importer (<< 0.9.14-1~), Multi-Arch: foreign -Description: Python modules for libloc - This package contains Python bindings for libloc +Description: ${source:Synopsis} (Python 3 bindings) + ${source:Extended-Description} + . + This package provides the Python 3 bindings for libloc. + +Package: location-python +Depends: python3-location, ${misc:Depends} +Architecture: all +Priority: optional +Section: oldlibs +Description: transitional package + This is a transitional package. It can safely be removed. + +Package: location-importer +Depends: location, ${misc:Depends} +Architecture: all +Priority: optional +Section: oldlibs +Description: transitional package + This is a transitional package. It can safely be removed. diff --git a/debian/copyright b/debian/copyright index 3bd7654..2877361 100644 --- a/debian/copyright +++ b/debian/copyright @@ -4,14 +4,516 @@ Upstream-Contact: Michael Tremer Source: https://location.ipfire.org/download Files: * -Copyright: 2017-2019 IPFire Development team -License: LGPL-2.1 +Copyright: 2017-2022, IPFire Development Team +License: LGPL-2.1+ + +Files: m4/* + src/test-address.c + src/test-as.c + src/test-country.c + src/test-database.c + src/test-libloc.c + src/test-network-list.c + src/test-network.c + src/test-signature.c + src/test-stringpool.c +Copyright: 2006-2008, Diego Pettenò + 2017-2022, IPFire Development Team + 2012, Lucas De Marchi + 2006-2008, xine project +License: GPL-2+ + +Files: src/perl/lib/* +Copyright: 2019, Stefan Schantl +License: Artistic-or-GPL + +Files: m4/ax_prog_perl_modules.m4 +Copyright: 2009, Dean Povey +License: FSFAP + +Files: m4/ld-version-script.m4 +Copyright: 2008-2015, Free Software Foundation, Inc +License: FSFULLR + +Files: tests/data/* +Copyright: 2017-2022, IPFire Development Team +License: CC-BY-SA-4 Files: debian/* -Copyright: 2019 Stefan Schantl -License: LGPL-2.1 +Copyright: 2022, Jochen Sprickerhof + 2019, Stefan Schantl +License: LGPL-2.1+ + +License: Artistic-or-GPL + This library is free software; you can redistribute it and/or modify + it under the same terms as Perl itself, either Perl version 5.28.1 or, + at your option, any later version of Perl 5 you may have available. + . + On Debian GNU/Linux systems, the complete text of the GNU General + Public License can be found in '/usr/share/common-licenses/GPL' and + the Artistic Licence in '/usr/share/common-licenses/Artistic'. -License: LGPL-2.1 +License: CC-BY-SA-4 + http://creativecommons.org/licenses/by-sa/4.0/ + . + Attribution-ShareAlike 4.0 International + . + ======================================================================= + . + Creative Commons Corporation ("Creative Commons") is not a law firm and + does not provide legal services or legal advice. Distribution of + Creative Commons public licenses does not create a lawyer-client or + other relationship. Creative Commons makes its licenses and related + information available on an "as-is" basis. Creative Commons gives no + warranties regarding its licenses, any material licensed under their + terms and conditions, or any related information. Creative Commons + disclaims all liability for damages resulting from their use to the + fullest extent possible. + . + Using Creative Commons Public Licenses + . + Creative Commons public licenses provide a standard set of terms and + conditions that creators and other rights holders may use to share + original works of authorship and other material subject to copyright + and certain other rights specified in the public license below. The + following considerations are for informational purposes only, are not + exhaustive, and do not form part of our licenses. + . + Considerations for licensors: Our public licenses are + intended for use by those authorized to give the public + permission to use material in ways otherwise restricted by + copyright and certain other rights. Our licenses are + irrevocable. Licensors should read and understand the terms + and conditions of the license they choose before applying it. + Licensors should also secure all rights necessary before + applying our licenses so that the public can reuse the + material as expected. Licensors should clearly mark any + material not subject to the license. This includes other CC- + licensed material, or material used under an exception or + limitation to copyright. More considerations for licensors: + wiki.creativecommons.org/Considerations_for_licensors + . + Considerations for the public: By using one of our public + licenses, a licensor grants the public permission to use the + licensed material under specified terms and conditions. If + the licensor's permission is not necessary for any reason--for + example, because of any applicable exception or limitation to + copyright--then that use is not regulated by the license. Our + licenses grant only permissions under copyright and certain + other rights that a licensor has authority to grant. Use of + the licensed material may still be restricted for other + reasons, including because others have copyright or other + rights in the material. A licensor may make special requests, + such as asking that all changes be marked or described. + Although not required by our licenses, you are encouraged to + respect those requests where reasonable. More_considerations + for the public: + wiki.creativecommons.org/Considerations_for_licensees + . + ======================================================================= + . + Creative Commons Attribution-ShareAlike 4.0 International Public + License + . + By exercising the Licensed Rights (defined below), You accept and agree + to be bound by the terms and conditions of this Creative Commons + Attribution-ShareAlike 4.0 International Public License ("Public + License"). To the extent this Public License may be interpreted as a + contract, You are granted the Licensed Rights in consideration of Your + acceptance of these terms and conditions, and the Licensor grants You + such rights in consideration of benefits the Licensor receives from + making the Licensed Material available under these terms and + conditions. + . + . + Section 1 -- Definitions. + . + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + . + b. Adapter's License means the license You apply to Your Copyright + and Similar Rights in Your contributions to Adapted Material in + accordance with the terms and conditions of this Public License. + . + c. BY-SA Compatible License means a license listed at + creativecommons.org/compatiblelicenses, approved by Creative + Commons as essentially the equivalent of this Public License. + . + d. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + . + e. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + . + f. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + . + g. License Elements means the license attributes listed in the name + of a Creative Commons Public License. The License Elements of this + Public License are Attribution and ShareAlike. + . + h. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + . + i. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + . + j. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + . + k. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + . + l. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + . + m. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + . + . + Section 2 -- Scope. + . + a. License grant. + . + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + . + a. reproduce and Share the Licensed Material, in whole or + in part; and + . + b. produce, reproduce, and Share Adapted Material. + . + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + . + 3. Term. The term of this Public License is specified in Section + 6(a). + . + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + . + 5. Downstream recipients. + . + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + . + b. Additional offer from the Licensor -- Adapted Material. + Every recipient of Adapted Material from You + automatically receives an offer from the Licensor to + exercise the Licensed Rights in the Adapted Material + under the conditions of the Adapter's License You apply. + . + c. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + . + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + . + b. Other rights. + . + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + . + 2. Patent and trademark rights are not licensed under this + Public License. + . + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties. + . + . + Section 3 -- License Conditions. + . + Your exercise of the Licensed Rights is expressly made subject to the + following conditions. + . + a. Attribution. + . + 1. If You Share the Licensed Material (including in modified + form), You must: + . + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + . + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + . + ii. a copyright notice; + . + iii. a notice that refers to this Public License; + . + iv. a notice that refers to the disclaimer of + warranties; + . + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + . + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + . + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + . + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + . + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + . + b. ShareAlike. + . + In addition to the conditions in Section 3(a), if You Share + Adapted Material You produce, the following conditions also apply. + . + 1. The Adapter's License You apply must be a Creative Commons + license with the same License Elements, this version or + later, or a BY-SA Compatible License. + . + 2. You must include the text of, or the URI or hyperlink to, the + Adapter's License You apply. You may satisfy this condition + in any reasonable manner based on the medium, means, and + context in which You Share Adapted Material. + . + 3. You may not offer or impose any additional or different terms + or conditions on, or apply any Effective Technological + Measures to, Adapted Material that restrict exercise of the + rights granted under the Adapter's License You apply. + . + . + Section 4 -- Sui Generis Database Rights. + . + Where the Licensed Rights include Sui Generis Database Rights that + apply to Your use of the Licensed Material: + . + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database; + . + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material, + . + including for purposes of Section 3(b); and + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + . + For the avoidance of doubt, this Section 4 supplements and does not + replace Your obligations under this Public License where the Licensed + Rights include other Copyright and Similar Rights. + . + . + Section 5 -- Disclaimer of Warranties and Limitation of Liability. + . + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + . + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + . + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + . + . + Section 6 -- Term and Termination. + . + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + . + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + . + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + . + 2. upon express reinstatement by the Licensor. + . + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + . + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + . + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + . + . + Section 7 -- Other Terms and Conditions. + . + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + . + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + . + . + Section 8 -- Interpretation. + . + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + . + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + . + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + . + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + . + . + ======================================================================= + . + Creative Commons is not a party to its public + licenses. Notwithstanding, Creative Commons may elect to apply one of + its public licenses to material it publishes and in those instances + will be considered the “Licensor.” The text of the Creative Commons + public licenses is dedicated to the public domain under the CC0 Public + Domain Dedication. Except for the limited purpose of indicating that + material is shared under a Creative Commons public license or as + otherwise permitted by the Creative Commons policies published at + creativecommons.org/policies, Creative Commons does not authorize the + use of the trademark "Creative Commons" or any other trademark or logo + of Creative Commons without its prior written consent including, + without limitation, in connection with any unauthorized modifications + to any of its public licenses or any other arrangements, + understandings, or agreements concerning use of licensed material. For + the avoidance of doubt, this paragraph does not form part of the + public licenses. + . + Creative Commons may be contacted at creativecommons.org. + +License: FSFAP + Copying and distribution of this file, with or without modification, are + permitted in any medium without royalty provided the copyright notice + and this notice are preserved. This file is offered as-is, without any + warranty. + +License: FSFULLR + This file is free software; the Free Software Foundation gives + unlimited permission to copy and/or distribute it, with or without + modifications, as long as this notice is preserved. + +License: GPL-2+ + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + . + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + . + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + . + On Debian systems, the complete text of the GNU General Public + License version 2 can be found in `/usr/share/common-licenses/GPL-2'. + +License: LGPL-2.1+ This program is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; version 2.1 of the License, or (at @@ -23,4 +525,3 @@ License: LGPL-2.1 . The complete text of the GNU General Public License can be found in /usr/share/common-licenses/LGPL-2.1 file. - . diff --git a/debian/libloc-dev.install b/debian/libloc-dev.install index d93d217..04e85fa 100644 --- a/debian/libloc-dev.install +++ b/debian/libloc-dev.install @@ -1,3 +1,4 @@ usr/include/libloc usr/lib/*/libloc.so usr/lib/*/pkgconfig +usr/share/man/man3 diff --git a/debian/libloc1.install b/debian/libloc1.install index 0f8eec4..e6cb2ac 100644 --- a/debian/libloc1.install +++ b/debian/libloc1.install @@ -1 +1,2 @@ usr/lib/*/libloc.so.* +usr/share/locale/*/LC_MESSAGES/libloc.mo diff --git a/debian/libloc1.symbols b/debian/libloc1.symbols index abdc32a..3770535 100644 --- a/debian/libloc1.symbols +++ b/debian/libloc1.symbols @@ -13,6 +13,7 @@ libloc.so.1 libloc1 #MINVER# loc_as_list_new@LIBLOC_1 0.9.5 loc_as_list_ref@LIBLOC_1 0.9.5 loc_as_list_size@LIBLOC_1 0.9.5 + loc_as_list_sort@LIBLOC_1 0.9.12 loc_as_list_unref@LIBLOC_1 0.9.5 loc_as_new@LIBLOC_1 0.9.4 loc_as_ref@LIBLOC_1 0.9.4 @@ -32,6 +33,7 @@ libloc.so.1 libloc1 #MINVER# loc_country_list_new@LIBLOC_1 0.9.5 loc_country_list_ref@LIBLOC_1 0.9.5 loc_country_list_size@LIBLOC_1 0.9.5 + loc_country_list_sort@LIBLOC_1 0.9.12 loc_country_list_unref@LIBLOC_1 0.9.5 loc_country_new@LIBLOC_1 0.9.4 loc_country_ref@LIBLOC_1 0.9.4 diff --git a/debian/location-importer.install b/debian/location-importer.install deleted file mode 100644 index eaae79d..0000000 --- a/debian/location-importer.install +++ /dev/null @@ -1,3 +0,0 @@ -usr/bin/location-importer -usr/lib/python3*/site-packages/location/database.py -usr/lib/python3*/site-packages/location/importer.py diff --git a/debian/location-perl.install b/debian/location-perl.install deleted file mode 100644 index 08e8cc4..0000000 --- a/debian/location-perl.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/*/perl/ diff --git a/debian/location-python.install b/debian/location-python.install deleted file mode 100644 index a6004ca..0000000 --- a/debian/location-python.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/python3*/site-packages diff --git a/debian/location.install b/debian/location.install index 25d9b6f..f9cb894 100644 --- a/debian/location.install +++ b/debian/location.install @@ -1,4 +1,4 @@ -usr/bin/location +usr/bin var/lib/location/signing-key.pem -src/systemd/*.service /lib/systemd/system/ -src/systemd/*.timer /lib/systemd/system/ +lib/systemd/system +usr/share/man/man8 diff --git a/debian/location.manpages b/debian/location.manpages deleted file mode 100644 index 3e662bb..0000000 --- a/debian/location.manpages +++ /dev/null @@ -1 +0,0 @@ -man/location.8 diff --git a/debian/location.postinst b/debian/location.postinst new file mode 100644 index 0000000..913f39c --- /dev/null +++ b/debian/location.postinst @@ -0,0 +1,14 @@ +#!/bin/sh + +set -e + +case "$1" in + configure) + mkdir -p /var/lib/location || true + ln -s /usr/share/libloc-location/location.db /var/lib/location/database.db 2>/dev/null || true + ;; +esac + +#DEBHELPER# + +exit 0 diff --git a/debian/location.postrm b/debian/location.postrm new file mode 100644 index 0000000..df1b03e --- /dev/null +++ b/debian/location.postrm @@ -0,0 +1,15 @@ +#!/bin/sh + +set -e + +case "$1" in + purge) + rm -f /var/lib/location/database.db 2>/dev/null + rm -f /var/lib/location/signing-key.pem 2>/dev/null + rmdir /var/lib/location || true + ;; +esac + +#DEBHELPER# + +exit 0 diff --git a/debian/location-python.examples b/debian/python3-location.examples similarity index 100% rename from debian/location-python.examples rename to debian/python3-location.examples diff --git a/debian/python3-location.install b/debian/python3-location.install new file mode 100644 index 0000000..4606faa --- /dev/null +++ b/debian/python3-location.install @@ -0,0 +1 @@ +usr/lib/python3* diff --git a/debian/rules b/debian/rules index 05b88fd..e5e3f18 100755 --- a/debian/rules +++ b/debian/rules @@ -1,28 +1,17 @@ #!/usr/bin/make -f -# enable verbose mode -#export DH_VERBOSE=1 - -# enable all hardening build flags export DEB_BUILD_MAINT_OPTIONS=hardening=+all +export PYBUILD_SYSTEM=custom +export PYBUILD_CLEAN_ARGS=dh_auto_clean +export PYBUILD_CONFIGURE_ARGS=intltoolize --force --automake; \ + PYTHON={interpreter} dh_auto_configure -- \ + --disable-perl +export PYBUILD_BUILD_ARGS=dh_auto_build +export PYBUILD_INSTALL_ARGS=dh_auto_install --destdir={destdir}; \ + mkdir -p {destdir}/usr/lib/python{version}/dist-packages; \ + mv {destdir}/usr/lib/python3/dist-packages/_location.so {destdir}/usr/lib/python{version}/dist-packages/_location.so; \ + rm -f {destdir}/usr/lib/python3/dist-packages/_location.la {destdir}/usr/lib/*/libloc.la +export PYBUILD_TEST_ARGS=dh_auto_test %: - dh $@ --with python3 --with-systemd - -override_dh_auto_configure: - intltoolize --force --automake - dh_auto_configure -- --disable-perl - -override_dh_perl: - dh_perl -d - -override_dh_systemd_enable: - dh_systemd_enable location-update.timer - -override_dh_install: - dh_install - # lintian: unknown-file-in-python-module-directory - rm debian/location-python/usr/lib/python3*/site-packages/_location.la - # linitan: binaries-have-file-conflict (d/location-importer.install) - rm debian/location-python/usr/lib/python3*/site-packages/location/database.py - rm debian/location-python/usr/lib/python3*/site-packages/location/importer.py + dh $@ --buildsystem=pybuild diff --git a/debian/watch b/debian/watch index 19ace6d..f466401 100644 --- a/debian/watch +++ b/debian/watch @@ -1,3 +1,3 @@ version=4 https://source.ipfire.org/releases/libloc/ \ - @PACKAGE@@ANY_VERSION@@ARCHIVE_EXT@ debian uupdate + @PACKAGE@@ANY_VERSION@@ARCHIVE_EXT@ diff --git a/po/POTFILES.in b/po/POTFILES.in index 5d2cc46..5f5afa8 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -1,12 +1,12 @@ src/libloc.pc.in -src/python/__init__.py.in -src/python/database.py -src/python/downloader.py -src/python/export.py -src/python/i18n.py -src/python/importer.py -src/python/location-importer.in -src/python/location.in -src/python/logger.py +src/python/location/__init__.py +src/python/location/database.py +src/python/location/downloader.py +src/python/location/export.py +src/python/location/i18n.py +src/python/location/importer.py +src/python/location/logger.py +src/scripts/location-importer.in +src/scripts/location.in src/systemd/location-update.service.in src/systemd/location-update.timer.in diff --git a/src/python/database.c b/src/python/database.c index ca56748..a33c31f 100644 --- a/src/python/database.c +++ b/src/python/database.c @@ -103,18 +103,24 @@ static PyObject* Database_verify(DatabaseObject* self, PyObject* args) { static PyObject* Database_get_description(DatabaseObject* self) { const char* description = loc_database_get_description(self->db); + if (!description) + Py_RETURN_NONE; return PyUnicode_FromString(description); } static PyObject* Database_get_vendor(DatabaseObject* self) { const char* vendor = loc_database_get_vendor(self->db); + if (!vendor) + Py_RETURN_NONE; return PyUnicode_FromString(vendor); } static PyObject* Database_get_license(DatabaseObject* self) { const char* license = loc_database_get_license(self->db); + if (!license) + Py_RETURN_NONE; return PyUnicode_FromString(license); } diff --git a/src/python/__init__.py.in b/src/python/location/__init__.py similarity index 98% rename from src/python/__init__.py.in rename to src/python/location/__init__.py index bd94d35..9b570c7 100644 --- a/src/python/__init__.py.in +++ b/src/python/location/__init__.py @@ -17,8 +17,6 @@ # # ############################################################################### -__version__ = "@VERSION@" - # Import everything from the C module from _location import * diff --git a/src/python/database.py b/src/python/location/database.py similarity index 100% rename from src/python/database.py rename to src/python/location/database.py diff --git a/src/python/downloader.py b/src/python/location/downloader.py similarity index 98% rename from src/python/downloader.py rename to src/python/location/downloader.py index 05f7872..b9e0c22 100644 --- a/src/python/downloader.py +++ b/src/python/location/downloader.py @@ -28,8 +28,7 @@ import urllib.error import urllib.parse import urllib.request -from . import __version__ -from _location import Database, DATABASE_VERSION_LATEST +from _location import Database, DATABASE_VERSION_LATEST, __version__ DATABASE_FILENAME = "location.db.xz" MIRRORS = ( diff --git a/src/python/export.py b/src/python/location/export.py similarity index 97% rename from src/python/export.py rename to src/python/location/export.py index 3cdece4..f5ed37f 100644 --- a/src/python/export.py +++ b/src/python/location/export.py @@ -17,7 +17,6 @@ # # ############################################################################### -import functools import io import ipaddress import logging @@ -59,6 +58,9 @@ class OutputWriter(object): else: self.f = io.StringIO() + # Tag + self.tag = self._make_tag() + # Call any custom initialization self.init() @@ -74,8 +76,7 @@ class OutputWriter(object): def __repr__(self): return "<%s %s f=%s>" % (self.__class__.__name__, self, self.f) - @functools.cached_property - def tag(self): + def _make_tag(self): families = { socket.AF_INET6 : "6", socket.AF_INET : "4", @@ -83,7 +84,7 @@ class OutputWriter(object): return "%sv%s" % (self.name, families.get(self.family, "?")) - @functools.cached_property + @property def filename(self): if self.directory: return os.path.join(self.directory, "%s.%s" % (self.tag, self.suffix)) @@ -180,7 +181,12 @@ class IpsetOutputWriter(OutputWriter): def _write_footer(self): # Jump back to the beginning of the file - self.f.seek(0) + try: + self.f.seek(0) + + # If the output stream isn't seekable, we won't try writing the header again + except io.UnsupportedOperation: + return # Rewrite the header with better configuration self._write_header() diff --git a/src/python/i18n.py b/src/python/location/i18n.py similarity index 100% rename from src/python/i18n.py rename to src/python/location/i18n.py diff --git a/src/python/importer.py b/src/python/location/importer.py similarity index 74% rename from src/python/importer.py rename to src/python/location/importer.py index dee36ed..96f2218 100644 --- a/src/python/importer.py +++ b/src/python/location/importer.py @@ -19,6 +19,7 @@ import gzip import logging +import tempfile import urllib.request # Initialise logging @@ -95,6 +96,9 @@ EXTENDED_SOURCES = { # ], } +# List all sources +SOURCES = set(WHOIS_SOURCES|EXTENDED_SOURCES) + class Downloader(object): def __init__(self): self.proxy = None @@ -106,75 +110,76 @@ class Downloader(object): log.info("Using proxy %s" % url) self.proxy = url - def request(self, url, data=None, return_blocks=False): + def retrieve(self, url, data=None): + """ + This method will fetch the content at the given URL + and will return a file-object to a temporary file. + + If the content was compressed, it will be decompressed on the fly. + """ + # Open a temporary file to buffer the downloaded content + t = tempfile.SpooledTemporaryFile(max_size=100 * 1024 * 1024) + + # Create a new request req = urllib.request.Request(url, data=data) # Configure proxy if self.proxy: req.set_proxy(self.proxy, "http") - return DownloaderContext(self, req, return_blocks=return_blocks) - - -class DownloaderContext(object): - def __init__(self, downloader, request, return_blocks=False): - self.downloader = downloader - self.request = request - - # Should we return one block or a single line? - self.return_blocks = return_blocks - - # Save the response object - self.response = None - - def __enter__(self): - log.info("Retrieving %s..." % self.request.full_url) + log.info("Retrieving %s..." % req.full_url) # Send request - self.response = urllib.request.urlopen(self.request) + res = urllib.request.urlopen(req) # Log the response headers log.debug("Response Headers:") - for header in self.headers: - log.debug(" %s: %s" % (header, self.get_header(header))) + for header in res.headers: + log.debug(" %s: %s" % (header, res.headers[header])) - return self + # Write the payload to the temporary file + with res as f: + while True: + buf = f.read(65536) + if not buf: + break - def __exit__(self, type, value, traceback): - pass + t.write(buf) - def __iter__(self): - """ - Makes the object iterable by going through each block - """ - if self.return_blocks: - return iterate_over_blocks(self.body) + # Rewind the temporary file + t.seek(0) - return iterate_over_lines(self.body) + # Fetch the content type + content_type = res.headers.get("Content-Type") - @property - def headers(self): - if self.response: - return self.response.headers + # Decompress any gzipped response on the fly + if content_type in ("application/x-gzip", "application/gzip"): + t = gzip.GzipFile(fileobj=t, mode="rb") - def get_header(self, name): - if self.headers: - return self.headers.get(name) + # Return the temporary file handle + return t - @property - def body(self): + def request_blocks(self, url, data=None): """ - Returns a file-like object with the decoded content - of the response. + This method will fetch the data from the URL and return an + iterator for each block in the data. """ - content_type = self.get_header("Content-Type") + # Download the data first + t = self.retrieve(url, data=data) - # Decompress any gzipped response on the fly - if content_type in ("application/x-gzip", "application/gzip"): - return gzip.GzipFile(fileobj=self.response, mode="rb") + # Then, split it into blocks + return iterate_over_blocks(t) + + def request_lines(self, url, data=None): + """ + This method will fetch the data from the URL and return an + iterator for each line in the data. + """ + # Download the data first + t = self.retrieve(url, data=data) - # Return the response by default - return self.response + # Then, split it into lines + return iterate_over_lines(t) def read_blocks(f): @@ -198,6 +203,10 @@ def iterate_over_blocks(f, charsets=("utf-8", "latin1")): block = [] for line in f: + # Skip commented lines + if line.startswith(b"#") or line.startswith(b"%"): + continue + # Convert to string for charset in charsets: try: @@ -207,24 +216,17 @@ def iterate_over_blocks(f, charsets=("utf-8", "latin1")): else: break - # Skip commented lines - if line.startswith("#") or line.startswith("%"): - continue - - # Strip line-endings - line = line.rstrip() - # Remove any comments at the end of line line, hash, comment = line.partition("#") - if comment: - # Strip any whitespace before the comment - line = line.rstrip() + # Strip any whitespace at the end of the line + line = line.rstrip() - # If the line is now empty, we move on - if not line: - continue + # If we cut off some comment and the line is empty, we can skip it + if comment and not line: + continue + # If the line has some content, keep collecting it if line: block.append(line) continue diff --git a/src/python/logger.py b/src/python/location/logger.py similarity index 100% rename from src/python/logger.py rename to src/python/location/logger.py diff --git a/src/python/locationmodule.c b/src/python/locationmodule.c index 15f661b..09cd5dd 100644 --- a/src/python/locationmodule.c +++ b/src/python/locationmodule.c @@ -117,6 +117,10 @@ PyMODINIT_FUNC PyInit__location(void) { if (!m) return NULL; + // Version + if (PyModule_AddStringConstant(m, "__version__", PACKAGE_VERSION)) + return NULL; + // AS if (PyType_Ready(&ASType) < 0) return NULL; diff --git a/src/python/location-importer.in b/src/scripts/location-importer.in similarity index 85% rename from src/python/location-importer.in rename to src/scripts/location-importer.in index bee9186..1785791 100644 --- a/src/python/location-importer.in +++ b/src/scripts/location-importer.in @@ -26,6 +26,7 @@ import re import socket import sys import telnetlib +import urllib.error # Load our location module import location @@ -403,127 +404,179 @@ class CLI(object): def handle_update_whois(self, ns): downloader = location.importer.Downloader() - # Download all sources - with self.db.transaction(): - # Create some temporary tables to store parsed data - self.db.execute(""" - CREATE TEMPORARY TABLE _autnums(number integer NOT NULL, organization text NOT NULL, source text NOT NULL) - ON COMMIT DROP; - CREATE UNIQUE INDEX _autnums_number ON _autnums(number); - - CREATE TEMPORARY TABLE _organizations(handle text NOT NULL, name text NOT NULL, source text NOT NULL) - ON COMMIT DROP; - CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle); - - CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL, original_countries text[] NOT NULL, source text NOT NULL) - ON COMMIT DROP; - CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network)); - CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network); - """) + # Did we run successfully? + error = False - # Remove all previously imported content - self.db.execute(""" - TRUNCATE TABLE networks; - """) + # Fetch all valid country codes to check parsed networks aganist + validcountries = self.countries - # Fetch all valid country codes to check parsed networks aganist... - rows = self.db.query("SELECT * FROM countries ORDER BY country_code") - validcountries = [] + # Iterate over all potential sources + for source in sorted(location.importer.SOURCES): + with self.db.transaction(): + # Create some temporary tables to store parsed data + self.db.execute(""" + CREATE TEMPORARY TABLE _autnums(number integer NOT NULL, + organization text NOT NULL, source text NOT NULL) ON COMMIT DROP; + CREATE UNIQUE INDEX _autnums_number ON _autnums(number); - for row in rows: - validcountries.append(row.country_code) + CREATE TEMPORARY TABLE _organizations(handle text NOT NULL, + name text NOT NULL, source text NOT NULL) ON COMMIT DROP; + CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle); - for source_key in location.importer.WHOIS_SOURCES: - for single_url in location.importer.WHOIS_SOURCES[source_key]: - with downloader.request(single_url, return_blocks=True) as f: - for block in f: - self._parse_block(block, source_key, validcountries) + CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL, + original_countries text[] NOT NULL, source text NOT NULL) + ON COMMIT DROP; + CREATE INDEX _rirdata_search ON _rirdata + USING BTREE(family(network), masklen(network)); + CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network); + """) - # Process all parsed networks from every RIR we happen to have access to, - # insert the largest network chunks into the networks table immediately... - families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)") + # Remove all previously imported content + self.db.execute("DELETE FROM networks WHERE source = %s", source) - for family in (row.family for row in families): - smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family) + try: + # Fetch WHOIS sources + for url in location.importer.WHOIS_SOURCES.get(source, []): + for block in downloader.request_blocks(url): + self._parse_block(block, source, validcountries) + + # Fetch extended sources + for url in location.importer.EXTENDED_SOURCES.get(source, []): + for line in downloader.request_lines(url): + self._parse_line(line, source, validcountries) + except urllib.error.URLError as e: + log.error("Could not retrieve data from %s: %s" % (source, e)) + error = True + + # Continue with the next source + continue - self.db.execute("INSERT INTO networks(network, country, original_countries, source) \ - SELECT network, country, original_countries, source FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family) + # Process all parsed networks from every RIR we happen to have access to, + # insert the largest network chunks into the networks table immediately... + families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata \ + ORDER BY family(network)") - # ... determine any other prefixes for this network family, ... - prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \ - WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family) + for family in (row.family for row in families): + # Fetch the smallest mask length in our data set + smallest = self.db.get(""" + SELECT + MIN( + masklen(network) + ) AS prefix + FROM + _rirdata + WHERE + family(network) = %s""", + family, + ) - # ... and insert networks with this prefix in case they provide additional - # information (i. e. subnet of a larger chunk with a different country) - for prefix in (row.prefix for row in prefixes): + # Copy all networks self.db.execute(""" - WITH candidates AS ( - SELECT - _rirdata.network, - _rirdata.country, - _rirdata.original_countries, - _rirdata.source - FROM - _rirdata - WHERE - family(_rirdata.network) = %s - AND - masklen(_rirdata.network) = %s - ), - filtered AS ( - SELECT - DISTINCT ON (c.network) - c.network, - c.country, - c.original_countries, - c.source, - masklen(networks.network), - networks.country AS parent_country - FROM - candidates c - LEFT JOIN - networks - ON - c.network << networks.network - ORDER BY - c.network, - masklen(networks.network) DESC NULLS LAST - ) INSERT INTO - networks(network, country, original_countries, source) + networks + ( + network, + country, + original_countries, + source + ) SELECT network, country, original_countries, source FROM - filtered + _rirdata WHERE - parent_country IS NULL - OR - country <> parent_country - ON CONFLICT DO NOTHING""", - family, prefix, + masklen(network) = %s + AND + family(network) = %s + ON CONFLICT DO + NOTHING""", + smallest.prefix, + family, ) - self.db.execute(""" - INSERT INTO autnums(number, name, source) - SELECT _autnums.number, _organizations.name, _organizations.source FROM _autnums - JOIN _organizations ON _autnums.organization = _organizations.handle - ON CONFLICT (number) DO UPDATE SET name = excluded.name; - """) + # ... determine any other prefixes for this network family, ... + prefixes = self.db.query(""" + SELECT + DISTINCT masklen(network) AS prefix + FROM + _rirdata + WHERE + family(network) = %s + ORDER BY + masklen(network) ASC + OFFSET 1""", + family, + ) - # Download all extended sources - for source_key in location.importer.EXTENDED_SOURCES: - for single_url in location.importer.EXTENDED_SOURCES[source_key]: - with self.db.transaction(): - # Download data - with downloader.request(single_url) as f: - for line in f: - self._parse_line(line, source_key, validcountries) + # ... and insert networks with this prefix in case they provide additional + # information (i. e. subnet of a larger chunk with a different country) + for prefix in (row.prefix for row in prefixes): + self.db.execute(""" + WITH candidates AS ( + SELECT + _rirdata.network, + _rirdata.country, + _rirdata.original_countries, + _rirdata.source + FROM + _rirdata + WHERE + family(_rirdata.network) = %s + AND + masklen(_rirdata.network) = %s + ), + filtered AS ( + SELECT + DISTINCT ON (c.network) + c.network, + c.country, + c.original_countries, + c.source, + masklen(networks.network), + networks.country AS parent_country + FROM + candidates c + LEFT JOIN + networks + ON + c.network << networks.network + ORDER BY + c.network, + masklen(networks.network) DESC NULLS LAST + ) + INSERT INTO + networks(network, country, original_countries, source) + SELECT + network, + country, + original_countries, + source + FROM + filtered + WHERE + parent_country IS NULL + OR + country <> parent_country + ON CONFLICT DO NOTHING""", + family, prefix, + ) + + self.db.execute(""" + INSERT INTO autnums(number, name, source) + SELECT _autnums.number, _organizations.name, _organizations.source FROM _autnums + JOIN _organizations ON _autnums.organization = _organizations.handle + ON CONFLICT (number) DO UPDATE SET name = excluded.name; + """) # Download and import (technical) AS names from ARIN - self._import_as_names_from_arin() + with self.db.transaction(): + self._import_as_names_from_arin() + + # Return a non-zero exit code for errors + return 1 if error else 0 def _check_parsed_network(self, network): """ @@ -871,50 +924,46 @@ class CLI(object): # technical, not intended for human consumption, as description fields in # organisation handles for other RIRs are - however, this is what we have got, # and in some cases, it might be still better than nothing) - with downloader.request("https://ftp.arin.net/info/asn.txt", return_blocks=False) as f: - for line in f: - # Convert binary line to string... - line = str(line) - - # ... valid lines start with a space, followed by the number of the Autonomous System ... - if not line.startswith(" "): - continue + for line in downloader.request_lines("https://ftp.arin.net/info/asn.txt"): + # Valid lines start with a space, followed by the number of the Autonomous System ... + if not line.startswith(" "): + continue - # Split line and check if there is a valid ASN in it... - asn, name = line.split()[0:2] + # Split line and check if there is a valid ASN in it... + asn, name = line.split()[0:2] - try: - asn = int(asn) - except ValueError: - log.debug("Skipping ARIN AS names line not containing an integer for ASN") - continue + try: + asn = int(asn) + except ValueError: + log.debug("Skipping ARIN AS names line not containing an integer for ASN") + continue - # Filter invalid ASNs... - if not self._check_parsed_asn(asn): - continue + # Filter invalid ASNs... + if not self._check_parsed_asn(asn): + continue - # Skip any AS name that appears to be a placeholder for a different RIR or entity... - if re.match(r"^(ASN-BLK|)(AFCONC|AFRINIC|APNIC|ASNBLK|DNIC|LACNIC|RIPE|IANA)(?:\d?$|\-)", name): - continue + # Skip any AS name that appears to be a placeholder for a different RIR or entity... + if re.match(r"^(ASN-BLK|)(AFCONC|AFRINIC|APNIC|ASNBLK|LACNIC|RIPE|IANA)(?:\d?$|\-)", name): + continue - # Bail out in case the AS name contains anything we do not expect here... - if re.search(r"[^a-zA-Z0-9-_]", name): - log.debug("Skipping ARIN AS name for %s containing invalid characters: %s" % \ - (asn, name)) + # Bail out in case the AS name contains anything we do not expect here... + if re.search(r"[^a-zA-Z0-9-_]", name): + log.debug("Skipping ARIN AS name for %s containing invalid characters: %s" % \ + (asn, name)) - # Things look good here, run INSERT statement and skip this one if we already have - # a (better?) name for this Autonomous System... - self.db.execute(""" - INSERT INTO autnums( - number, - name, - source - ) VALUES (%s, %s, %s) - ON CONFLICT (number) DO NOTHING""", - asn, + # Things look good here, run INSERT statement and skip this one if we already have + # a (better?) name for this Autonomous System... + self.db.execute(""" + INSERT INTO autnums( + number, name, - "ARIN", - ) + source + ) VALUES (%s, %s, %s) + ON CONFLICT (number) DO NOTHING""", + asn, + name, + "ARIN", + ) def handle_update_announcements(self, ns): server = ns.server[0] @@ -1168,10 +1217,11 @@ class CLI(object): def handle_update_overrides(self, ns): with self.db.transaction(): - # Drop all data that we have + # Only drop manually created overrides, as we can be reasonably sure to have them, + # and preserve the rest. If appropriate, it is deleted by correspondent functions. self.db.execute(""" - TRUNCATE TABLE autnum_overrides; - TRUNCATE TABLE network_overrides; + DELETE FROM autnum_overrides WHERE source = 'manual'; + DELETE FROM network_overrides WHERE source = 'manual'; """) # Update overrides for various cloud providers big enough to publish their own IP @@ -1261,12 +1311,20 @@ class CLI(object): downloader = location.importer.Downloader() try: - with downloader.request("https://ip-ranges.amazonaws.com/ip-ranges.json", return_blocks=False) as f: - aws_ip_dump = json.load(f.body) + # Fetch IP ranges + f = downloader.retrieve("https://ip-ranges.amazonaws.com/ip-ranges.json") + + # Parse downloaded file + aws_ip_dump = json.load(f) except Exception as e: log.error("unable to preprocess Amazon AWS IP ranges: %s" % e) return + # At this point, we can assume the downloaded file to be valid + self.db.execute(""" + DELETE FROM network_overrides WHERE source = 'Amazon AWS IP feed'; + """) + # XXX: Set up a dictionary for mapping a region name to a country. Unfortunately, # there seems to be no machine-readable version available of this other than # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html @@ -1380,18 +1438,27 @@ class CLI(object): ] for url in ip_urls: - try: - with downloader.request(url, return_blocks=False) as f: - fcontent = f.body.readlines() - except Exception as e: - log.error("Unable to download Spamhaus DROP URL %s: %s" % (url, e)) - return + # Fetch IP list + f = downloader.retrieve(url) + + # Split into lines + fcontent = f.readlines() + + # Conduct a very basic sanity check to rule out CDN issues causing bogus DROP + # downloads. + if len(fcontent) > 10: + self.db.execute(""" + DELETE FROM autnum_overrides WHERE source = 'Spamhaus ASN-DROP list'; + DELETE FROM network_overrides WHERE source = 'Spamhaus DROP lists'; + """) + else: + log.error("Spamhaus DROP URL %s returned likely bogus file, ignored" % url) + continue # Iterate through every line, filter comments and add remaining networks to # the override table in case they are valid... with self.db.transaction(): for sline in fcontent: - # The response is assumed to be encoded in UTF-8... sline = sline.decode("utf-8") @@ -1426,18 +1493,13 @@ class CLI(object): ) for url in asn_urls: - try: - with downloader.request(url, return_blocks=False) as f: - fcontent = f.body.readlines() - except Exception as e: - log.error("Unable to download Spamhaus DROP URL %s: %s" % (url, e)) - return + # Fetch URL + f = downloader.retrieve(url) # Iterate through every line, filter comments and add remaining ASNs to # the override table in case they are valid... with self.db.transaction(): - for sline in fcontent: - + for sline in t.readlines(): # The response is assumed to be encoded in UTF-8... sline = sline.decode("utf-8") @@ -1495,6 +1557,14 @@ class CLI(object): # Default to None return None + @property + def countries(self): + # Fetch all valid country codes to check parsed networks aganist + rows = self.db.query("SELECT * FROM countries ORDER BY country_code") + + # Return all countries + return [row.country_code for row in rows] + def handle_import_countries(self, ns): with self.db.transaction(): # Drop all data that we have diff --git a/src/python/location.in b/src/scripts/location.in similarity index 99% rename from src/python/location.in rename to src/scripts/location.in index 233cea0..25119a8 100644 --- a/src/python/location.in +++ b/src/scripts/location.in @@ -481,7 +481,7 @@ class CLI(object): return 1 # Success - log.debug("Database successfully verified") + log.info("Database successfully verified") return 0 def __get_output_formatter(self, ns): -- 2.47.2