/*.db.xz
/libtool
/stamp-h1
-/src/python/location
-/src/python/location-importer
-/src/python/__init__.py
+/src/scripts/location
+/src/scripts/location-importer
/src/systemd/location-update.service
/src/systemd/location-update.timer
/test.db
pkgconfigdir = $(libdir)/pkgconfig
# Overwrite Python path
-pkgpythondir = $(pythondir)/location
+#pkgpythondir = $(pythondir)/location
+pkgpythondir = /usr/lib/python3/dist-packages/location
%: %.in Makefile
$(SED_PROCESS)
src/libloc.pc
dist_pkgpython_PYTHON = \
- src/python/database.py \
- src/python/downloader.py \
- src/python/export.py \
- src/python/i18n.py \
- src/python/importer.py \
- src/python/logger.py
-
-pkgpython_PYTHON = \
- src/python/__init__.py
-
-EXTRA_DIST += \
- src/python/__init__.py.in
-
-CLEANFILES += \
- src/python/__init__.py
+ src/python/location/__init__.py \
+ src/python/location/database.py \
+ src/python/location/downloader.py \
+ src/python/location/export.py \
+ src/python/location/i18n.py \
+ src/python/location/importer.py \
+ src/python/location/logger.py
pyexec_LTLIBRARIES = \
src/python/_location.la
$(DESTDIR)/$(prefix)/man/man3/Location.3pm
bin_SCRIPTS = \
- src/python/location \
- src/python/location-importer
+ src/scripts/location \
+ src/scripts/location-importer
EXTRA_DIST += \
- src/python/location.in \
- src/python/location-importer.in
+ src/scripts/location.in \
+ src/scripts/location-importer.in
CLEANFILES += \
- src/python/location \
- src/python/location-importer
+ src/scripts/location \
+ src/scripts/location-importer
# ------------------------------------------------------------------------------
src/libloc-internal.la
TESTS_ENVIRONMENT = \
+ PYTHONPATH=$(abs_srcdir)/src/python:$(abs_builddir)/src/python/.libs \
TEST_DATA_DIR="$(abs_top_srcdir)/tests/data"
TESTS = \
testdata.db
testdata.db: examples/python/create-database.py
- PYTHONPATH=$(abs_builddir)/src/python/.libs \
+ PYTHONPATH=$(abs_srcdir)/src/python:$(abs_builddir)/src/python/.libs \
ABS_SRCDIR="$(abs_srcdir)" \
$(PYTHON) $< $@
EXTRA_DIST += \
debian/build.sh \
debian/changelog \
- debian/compat \
debian/control \
debian/copyright \
- debian/location.install \
- debian/location.manpages \
- debian/location-python.install \
+ debian/genchangelog.sh \
+ debian/gensymbols.sh \
debian/libloc1.install \
+ debian/libloc1.symbols \
debian/libloc-dev.install \
+ debian/location.install \
+ debian/location.postinst \
+ debian/location.postrm \
+ debian/python3-location.examples \
+ debian/python3-location.install \
debian/rules \
- debian/source/format
+ debian/source/format \
+ debian/watch
.PHONY: debian
debian: dist
AC_PREREQ(2.60)
AC_INIT([libloc],
- [0.9.13],
+ [0.9.14],
[location@lists.ipfire.org],
[libloc],
[https://location.ipfire.org/])
# ------------------------------------------------------------------------------
# Python
-AM_PATH_PYTHON([3.9])
+AM_PATH_PYTHON([3.4])
PKG_CHECK_MODULES([PYTHON], [python-${PYTHON_VERSION}])
# Perl
local release
for release in ${RELEASES[@]}; do
local chroot="${release}-${host_arch}-sbuild"
+ if [ "${release}" = "buster" ]; then
+ local buster_backport=( --extra-repository "deb http://deb.debian.org/debian buster-backports main" --build-dep-resolver=aspcud )
+ fi
mkdir -p "${release}"
pushd "${release}"
cp -r "${tmp}/sources" .
# Run the build process
- if ! sbuild --dist="${release}" --host="${arch}" --source "sources/${package}"; then
+ if ! sbuild --dist="${release}" --host="${arch}" --source "${buster_backport[@]}" "sources/${package}"; then
echo "Could not build package for ${release} on ${arch}" >&2
rm -rf "${tmp}"
return 1
+libloc (0.9.14-1) unstable; urgency=medium
+
+ [ Michael Tremer ]
+ * Revert "configure: Require Python >= 3.9"
+ * Make sources around that we can run tests without location installed
+ * downloader: Fetch __version__ from C module
+ * export: Drop using functools
+ * verify: Show message on success
+ * export: Don't fail when output stream isn't seekable
+ * importer: Actually perform the Spamhaus sanity check
+ * importer: Change download behaviour
+ * importer: Move importing extended sources/ARIN into transaction
+ * python: database: Return None if no description/vendor/license set
+ * importer: Try to make parsing blocks faster
+ * importer: Import each source individually
+ * python: Fix missing bracket
+ * importer: Tolerate that data might exist from other RIRs
+ * importer: Import all sources in alphabetical order
+
+ [ Peter Müller ]
+ * location-importer: Only delete override data if we are sure to have
+ a valid replacement
+ * location-importer: AS names starting with "DNIC" actually are valid
+
+ -- Michael Tremer <michael.tremer@ipfire.org> Sun, 14 Aug 2022 12:24:16 +0000
+
libloc (0.9.13-1) unstable; urgency=medium
[ Michael Tremer ]
Source: libloc
Maintainer: Stefan Schantl <stefan.schantl@ipfire.org>
-Section: misc
+Section: net
Priority: optional
-Standards-Version: 4.3.0
+Standards-Version: 4.6.1
Build-Depends:
- debhelper (>= 11),
- dh-python <!nopython>,
- asciidoc <!nodoc>,
- intltool (>=0.40.0),
- libpython3-dev <!nopython>,
+ debhelper-compat (= 13),
+ dh-sequence-python3,
+ asciidoc,
+ intltool,
libssl-dev,
libsystemd-dev,
- python3-dev:any <!nopython>,
pkg-config,
+ python3-all-dev,
systemd,
- xsltproc <!nodoc>,
- docbook-xsl <!nodoc>,
- git,
+ xsltproc,
+ docbook-xsl,
Rules-Requires-Root: no
Homepage: https://location.ipfire.org/
-Vcs-Git: https://git.ipfire.org/pub/git/location/libloc.git
-Vcs-Browser: https://git.ipfire.org/pub/git/location/libloc.git
+Vcs-Git: https://salsa.debian.org/debian/libloc.git
+Vcs-Browser: https://salsa.debian.org/debian/libloc
+Description: IP geolocation query library
+ libloc is a lightweight library to query the IPFire Location database and
+ determine the location of someone else on the Internet based on their IP
+ address.
Package: libloc1
Architecture: any
Section: libs
-Pre-Depends:
- ${misc:Pre-Depends}
Depends:
${shlibs:Depends},
- ${misc:Depends}
-Recommends:
- location (= ${binary:Version})
+ ${misc:Depends},
Multi-Arch: same
-Description: Location library
- A library to determine the location of someone on the Internet
+Description: ${source:Synopsis}
+ ${source:Extended-Description}
+ .
+ This package provides the shared library.
Package: libloc-dev
Architecture: any
Depends:
libloc1 (= ${binary:Version}),
${misc:Depends},
-Suggests:
- pkg-config
Multi-Arch: same
-Description: Development files for libloc
- Install this package if you wish to develop your own programs using
- libloc.
+Description: ${source:Synopsis} (development files)
+ ${source:Extended-Description}
+ .
+ This package provides the headers and development files needed to use libloc
+ in your own programs.
Package: location
-Architecture: any
-Pre-Depends:
- ${misc:Pre-Depends}
-Depends:
- location-python (= ${binary:Version}),
- ${misc:Depends},
- ${python3:Depends}
-Multi-Arch: same
-Description: CLI utilities for libloc
- Commands to determine someone's location on the Internet
-
-Package: location-importer
-Architecture: any
-Pre-Depends:
- ${misc:Pre-Depends}
+Architecture: all
Depends:
- location-python (= ${binary:Version}),
+ python3-location,
${misc:Depends},
- ${python3:Depends}
-Multi-Arch: foreign
-Description: Tools to author location databases
- This package contains tools that are required to build location databases
+ ${python3:Depends},
+Recommends:
+ libloc-database,
+Replaces: location-importer (<< 0.9.14-1~)
+Breaks: location-importer (<< 0.9.14-1~)
+Description: ${source:Synopsis} (CLI utilities)
+ ${source:Extended-Description}
+ .
+ This package provides CLI utilities based on libloc.
-Package: location-python
+Package: python3-location
Architecture: any
Section: python
-Pre-Depends:
- ${misc:Pre-Depends}
Depends:
${misc:Depends},
${python3:Depends},
- ${shlibs:Depends}
+ ${shlibs:Depends},
+ python3-psycopg2,
+Replaces:
+ location-python (<< 0.9.14-1~),
+Breaks:
+ location-python (<< 0.9.14-1~),
+ location-importer (<< 0.9.14-1~),
Multi-Arch: foreign
-Description: Python modules for libloc
- This package contains Python bindings for libloc
+Description: ${source:Synopsis} (Python 3 bindings)
+ ${source:Extended-Description}
+ .
+ This package provides the Python 3 bindings for libloc.
+
+Package: location-python
+Depends: python3-location, ${misc:Depends}
+Architecture: all
+Priority: optional
+Section: oldlibs
+Description: transitional package
+ This is a transitional package. It can safely be removed.
+
+Package: location-importer
+Depends: location, ${misc:Depends}
+Architecture: all
+Priority: optional
+Section: oldlibs
+Description: transitional package
+ This is a transitional package. It can safely be removed.
Source: https://location.ipfire.org/download
Files: *
-Copyright: 2017-2019 IPFire Development team <info@ipfire.org>
-License: LGPL-2.1
+Copyright: 2017-2022, IPFire Development Team <info@ipfire.org>
+License: LGPL-2.1+
+
+Files: m4/*
+ src/test-address.c
+ src/test-as.c
+ src/test-country.c
+ src/test-database.c
+ src/test-libloc.c
+ src/test-network-list.c
+ src/test-network.c
+ src/test-signature.c
+ src/test-stringpool.c
+Copyright: 2006-2008, Diego Pettenò <flameeyes@gmail.com>
+ 2017-2022, IPFire Development Team <info@ipfire.org>
+ 2012, Lucas De Marchi <lucas.de.marchi@gmail.com>
+ 2006-2008, xine project
+License: GPL-2+
+
+Files: src/perl/lib/*
+Copyright: 2019, Stefan Schantl
+License: Artistic-or-GPL
+
+Files: m4/ax_prog_perl_modules.m4
+Copyright: 2009, Dean Povey <povey@wedgetail.com>
+License: FSFAP
+
+Files: m4/ld-version-script.m4
+Copyright: 2008-2015, Free Software Foundation, Inc
+License: FSFULLR
+
+Files: tests/data/*
+Copyright: 2017-2022, IPFire Development Team <info@ipfire.org>
+License: CC-BY-SA-4
Files: debian/*
-Copyright: 2019 Stefan Schantl <stefan.schantl@ipfire.org>
-License: LGPL-2.1
+Copyright: 2022, Jochen Sprickerhof <jspricke@debian.org>
+ 2019, Stefan Schantl <stefan.schantl@ipfire.org>
+License: LGPL-2.1+
+
+License: Artistic-or-GPL
+ This library is free software; you can redistribute it and/or modify
+ it under the same terms as Perl itself, either Perl version 5.28.1 or,
+ at your option, any later version of Perl 5 you may have available.
+ .
+ On Debian GNU/Linux systems, the complete text of the GNU General
+ Public License can be found in '/usr/share/common-licenses/GPL' and
+ the Artistic Licence in '/usr/share/common-licenses/Artistic'.
-License: LGPL-2.1
+License: CC-BY-SA-4
+ http://creativecommons.org/licenses/by-sa/4.0/
+ .
+ Attribution-ShareAlike 4.0 International
+ .
+ =======================================================================
+ .
+ Creative Commons Corporation ("Creative Commons") is not a law firm and
+ does not provide legal services or legal advice. Distribution of
+ Creative Commons public licenses does not create a lawyer-client or
+ other relationship. Creative Commons makes its licenses and related
+ information available on an "as-is" basis. Creative Commons gives no
+ warranties regarding its licenses, any material licensed under their
+ terms and conditions, or any related information. Creative Commons
+ disclaims all liability for damages resulting from their use to the
+ fullest extent possible.
+ .
+ Using Creative Commons Public Licenses
+ .
+ Creative Commons public licenses provide a standard set of terms and
+ conditions that creators and other rights holders may use to share
+ original works of authorship and other material subject to copyright
+ and certain other rights specified in the public license below. The
+ following considerations are for informational purposes only, are not
+ exhaustive, and do not form part of our licenses.
+ .
+ Considerations for licensors: Our public licenses are
+ intended for use by those authorized to give the public
+ permission to use material in ways otherwise restricted by
+ copyright and certain other rights. Our licenses are
+ irrevocable. Licensors should read and understand the terms
+ and conditions of the license they choose before applying it.
+ Licensors should also secure all rights necessary before
+ applying our licenses so that the public can reuse the
+ material as expected. Licensors should clearly mark any
+ material not subject to the license. This includes other CC-
+ licensed material, or material used under an exception or
+ limitation to copyright. More considerations for licensors:
+ wiki.creativecommons.org/Considerations_for_licensors
+ .
+ Considerations for the public: By using one of our public
+ licenses, a licensor grants the public permission to use the
+ licensed material under specified terms and conditions. If
+ the licensor's permission is not necessary for any reason--for
+ example, because of any applicable exception or limitation to
+ copyright--then that use is not regulated by the license. Our
+ licenses grant only permissions under copyright and certain
+ other rights that a licensor has authority to grant. Use of
+ the licensed material may still be restricted for other
+ reasons, including because others have copyright or other
+ rights in the material. A licensor may make special requests,
+ such as asking that all changes be marked or described.
+ Although not required by our licenses, you are encouraged to
+ respect those requests where reasonable. More_considerations
+ for the public:
+ wiki.creativecommons.org/Considerations_for_licensees
+ .
+ =======================================================================
+ .
+ Creative Commons Attribution-ShareAlike 4.0 International Public
+ License
+ .
+ By exercising the Licensed Rights (defined below), You accept and agree
+ to be bound by the terms and conditions of this Creative Commons
+ Attribution-ShareAlike 4.0 International Public License ("Public
+ License"). To the extent this Public License may be interpreted as a
+ contract, You are granted the Licensed Rights in consideration of Your
+ acceptance of these terms and conditions, and the Licensor grants You
+ such rights in consideration of benefits the Licensor receives from
+ making the Licensed Material available under these terms and
+ conditions.
+ .
+ .
+ Section 1 -- Definitions.
+ .
+ a. Adapted Material means material subject to Copyright and Similar
+ Rights that is derived from or based upon the Licensed Material
+ and in which the Licensed Material is translated, altered,
+ arranged, transformed, or otherwise modified in a manner requiring
+ permission under the Copyright and Similar Rights held by the
+ Licensor. For purposes of this Public License, where the Licensed
+ Material is a musical work, performance, or sound recording,
+ Adapted Material is always produced where the Licensed Material is
+ synched in timed relation with a moving image.
+ .
+ b. Adapter's License means the license You apply to Your Copyright
+ and Similar Rights in Your contributions to Adapted Material in
+ accordance with the terms and conditions of this Public License.
+ .
+ c. BY-SA Compatible License means a license listed at
+ creativecommons.org/compatiblelicenses, approved by Creative
+ Commons as essentially the equivalent of this Public License.
+ .
+ d. Copyright and Similar Rights means copyright and/or similar rights
+ closely related to copyright including, without limitation,
+ performance, broadcast, sound recording, and Sui Generis Database
+ Rights, without regard to how the rights are labeled or
+ categorized. For purposes of this Public License, the rights
+ specified in Section 2(b)(1)-(2) are not Copyright and Similar
+ Rights.
+ .
+ e. Effective Technological Measures means those measures that, in the
+ absence of proper authority, may not be circumvented under laws
+ fulfilling obligations under Article 11 of the WIPO Copyright
+ Treaty adopted on December 20, 1996, and/or similar international
+ agreements.
+ .
+ f. Exceptions and Limitations means fair use, fair dealing, and/or
+ any other exception or limitation to Copyright and Similar Rights
+ that applies to Your use of the Licensed Material.
+ .
+ g. License Elements means the license attributes listed in the name
+ of a Creative Commons Public License. The License Elements of this
+ Public License are Attribution and ShareAlike.
+ .
+ h. Licensed Material means the artistic or literary work, database,
+ or other material to which the Licensor applied this Public
+ License.
+ .
+ i. Licensed Rights means the rights granted to You subject to the
+ terms and conditions of this Public License, which are limited to
+ all Copyright and Similar Rights that apply to Your use of the
+ Licensed Material and that the Licensor has authority to license.
+ .
+ j. Licensor means the individual(s) or entity(ies) granting rights
+ under this Public License.
+ .
+ k. Share means to provide material to the public by any means or
+ process that requires permission under the Licensed Rights, such
+ as reproduction, public display, public performance, distribution,
+ dissemination, communication, or importation, and to make material
+ available to the public including in ways that members of the
+ public may access the material from a place and at a time
+ individually chosen by them.
+ .
+ l. Sui Generis Database Rights means rights other than copyright
+ resulting from Directive 96/9/EC of the European Parliament and of
+ the Council of 11 March 1996 on the legal protection of databases,
+ as amended and/or succeeded, as well as other essentially
+ equivalent rights anywhere in the world.
+ .
+ m. You means the individual or entity exercising the Licensed Rights
+ under this Public License. Your has a corresponding meaning.
+ .
+ .
+ Section 2 -- Scope.
+ .
+ a. License grant.
+ .
+ 1. Subject to the terms and conditions of this Public License,
+ the Licensor hereby grants You a worldwide, royalty-free,
+ non-sublicensable, non-exclusive, irrevocable license to
+ exercise the Licensed Rights in the Licensed Material to:
+ .
+ a. reproduce and Share the Licensed Material, in whole or
+ in part; and
+ .
+ b. produce, reproduce, and Share Adapted Material.
+ .
+ 2. Exceptions and Limitations. For the avoidance of doubt, where
+ Exceptions and Limitations apply to Your use, this Public
+ License does not apply, and You do not need to comply with
+ its terms and conditions.
+ .
+ 3. Term. The term of this Public License is specified in Section
+ 6(a).
+ .
+ 4. Media and formats; technical modifications allowed. The
+ Licensor authorizes You to exercise the Licensed Rights in
+ all media and formats whether now known or hereafter created,
+ and to make technical modifications necessary to do so. The
+ Licensor waives and/or agrees not to assert any right or
+ authority to forbid You from making technical modifications
+ necessary to exercise the Licensed Rights, including
+ technical modifications necessary to circumvent Effective
+ Technological Measures. For purposes of this Public License,
+ simply making modifications authorized by this Section 2(a)
+ (4) never produces Adapted Material.
+ .
+ 5. Downstream recipients.
+ .
+ a. Offer from the Licensor -- Licensed Material. Every
+ recipient of the Licensed Material automatically
+ receives an offer from the Licensor to exercise the
+ Licensed Rights under the terms and conditions of this
+ Public License.
+ .
+ b. Additional offer from the Licensor -- Adapted Material.
+ Every recipient of Adapted Material from You
+ automatically receives an offer from the Licensor to
+ exercise the Licensed Rights in the Adapted Material
+ under the conditions of the Adapter's License You apply.
+ .
+ c. No downstream restrictions. You may not offer or impose
+ any additional or different terms or conditions on, or
+ apply any Effective Technological Measures to, the
+ Licensed Material if doing so restricts exercise of the
+ Licensed Rights by any recipient of the Licensed
+ Material.
+ .
+ 6. No endorsement. Nothing in this Public License constitutes or
+ may be construed as permission to assert or imply that You
+ are, or that Your use of the Licensed Material is, connected
+ with, or sponsored, endorsed, or granted official status by,
+ the Licensor or others designated to receive attribution as
+ provided in Section 3(a)(1)(A)(i).
+ .
+ b. Other rights.
+ .
+ 1. Moral rights, such as the right of integrity, are not
+ licensed under this Public License, nor are publicity,
+ privacy, and/or other similar personality rights; however, to
+ the extent possible, the Licensor waives and/or agrees not to
+ assert any such rights held by the Licensor to the limited
+ extent necessary to allow You to exercise the Licensed
+ Rights, but not otherwise.
+ .
+ 2. Patent and trademark rights are not licensed under this
+ Public License.
+ .
+ 3. To the extent possible, the Licensor waives any right to
+ collect royalties from You for the exercise of the Licensed
+ Rights, whether directly or through a collecting society
+ under any voluntary or waivable statutory or compulsory
+ licensing scheme. In all other cases the Licensor expressly
+ reserves any right to collect such royalties.
+ .
+ .
+ Section 3 -- License Conditions.
+ .
+ Your exercise of the Licensed Rights is expressly made subject to the
+ following conditions.
+ .
+ a. Attribution.
+ .
+ 1. If You Share the Licensed Material (including in modified
+ form), You must:
+ .
+ a. retain the following if it is supplied by the Licensor
+ with the Licensed Material:
+ .
+ i. identification of the creator(s) of the Licensed
+ Material and any others designated to receive
+ attribution, in any reasonable manner requested by
+ the Licensor (including by pseudonym if
+ designated);
+ .
+ ii. a copyright notice;
+ .
+ iii. a notice that refers to this Public License;
+ .
+ iv. a notice that refers to the disclaimer of
+ warranties;
+ .
+ v. a URI or hyperlink to the Licensed Material to the
+ extent reasonably practicable;
+ .
+ b. indicate if You modified the Licensed Material and
+ retain an indication of any previous modifications; and
+ .
+ c. indicate the Licensed Material is licensed under this
+ Public License, and include the text of, or the URI or
+ hyperlink to, this Public License.
+ .
+ 2. You may satisfy the conditions in Section 3(a)(1) in any
+ reasonable manner based on the medium, means, and context in
+ which You Share the Licensed Material. For example, it may be
+ reasonable to satisfy the conditions by providing a URI or
+ hyperlink to a resource that includes the required
+ information.
+ .
+ 3. If requested by the Licensor, You must remove any of the
+ information required by Section 3(a)(1)(A) to the extent
+ reasonably practicable.
+ .
+ b. ShareAlike.
+ .
+ In addition to the conditions in Section 3(a), if You Share
+ Adapted Material You produce, the following conditions also apply.
+ .
+ 1. The Adapter's License You apply must be a Creative Commons
+ license with the same License Elements, this version or
+ later, or a BY-SA Compatible License.
+ .
+ 2. You must include the text of, or the URI or hyperlink to, the
+ Adapter's License You apply. You may satisfy this condition
+ in any reasonable manner based on the medium, means, and
+ context in which You Share Adapted Material.
+ .
+ 3. You may not offer or impose any additional or different terms
+ or conditions on, or apply any Effective Technological
+ Measures to, Adapted Material that restrict exercise of the
+ rights granted under the Adapter's License You apply.
+ .
+ .
+ Section 4 -- Sui Generis Database Rights.
+ .
+ Where the Licensed Rights include Sui Generis Database Rights that
+ apply to Your use of the Licensed Material:
+ .
+ a. for the avoidance of doubt, Section 2(a)(1) grants You the right
+ to extract, reuse, reproduce, and Share all or a substantial
+ portion of the contents of the database;
+ .
+ b. if You include all or a substantial portion of the database
+ contents in a database in which You have Sui Generis Database
+ Rights, then the database in which You have Sui Generis Database
+ Rights (but not its individual contents) is Adapted Material,
+ .
+ including for purposes of Section 3(b); and
+ c. You must comply with the conditions in Section 3(a) if You Share
+ all or a substantial portion of the contents of the database.
+ .
+ For the avoidance of doubt, this Section 4 supplements and does not
+ replace Your obligations under this Public License where the Licensed
+ Rights include other Copyright and Similar Rights.
+ .
+ .
+ Section 5 -- Disclaimer of Warranties and Limitation of Liability.
+ .
+ a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
+ EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
+ AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
+ ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
+ IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
+ WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
+ ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
+ KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
+ ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
+ .
+ b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
+ TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
+ NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
+ INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
+ COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
+ USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
+ ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
+ DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
+ IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
+ .
+ c. The disclaimer of warranties and limitation of liability provided
+ above shall be interpreted in a manner that, to the extent
+ possible, most closely approximates an absolute disclaimer and
+ waiver of all liability.
+ .
+ .
+ Section 6 -- Term and Termination.
+ .
+ a. This Public License applies for the term of the Copyright and
+ Similar Rights licensed here. However, if You fail to comply with
+ this Public License, then Your rights under this Public License
+ terminate automatically.
+ .
+ b. Where Your right to use the Licensed Material has terminated under
+ Section 6(a), it reinstates:
+ .
+ 1. automatically as of the date the violation is cured, provided
+ it is cured within 30 days of Your discovery of the
+ violation; or
+ .
+ 2. upon express reinstatement by the Licensor.
+ .
+ For the avoidance of doubt, this Section 6(b) does not affect any
+ right the Licensor may have to seek remedies for Your violations
+ of this Public License.
+ .
+ c. For the avoidance of doubt, the Licensor may also offer the
+ Licensed Material under separate terms or conditions or stop
+ distributing the Licensed Material at any time; however, doing so
+ will not terminate this Public License.
+ .
+ d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
+ License.
+ .
+ .
+ Section 7 -- Other Terms and Conditions.
+ .
+ a. The Licensor shall not be bound by any additional or different
+ terms or conditions communicated by You unless expressly agreed.
+ .
+ b. Any arrangements, understandings, or agreements regarding the
+ Licensed Material not stated herein are separate from and
+ independent of the terms and conditions of this Public License.
+ .
+ .
+ Section 8 -- Interpretation.
+ .
+ a. For the avoidance of doubt, this Public License does not, and
+ shall not be interpreted to, reduce, limit, restrict, or impose
+ conditions on any use of the Licensed Material that could lawfully
+ be made without permission under this Public License.
+ .
+ b. To the extent possible, if any provision of this Public License is
+ deemed unenforceable, it shall be automatically reformed to the
+ minimum extent necessary to make it enforceable. If the provision
+ cannot be reformed, it shall be severed from this Public License
+ without affecting the enforceability of the remaining terms and
+ conditions.
+ .
+ c. No term or condition of this Public License will be waived and no
+ failure to comply consented to unless expressly agreed to by the
+ Licensor.
+ .
+ d. Nothing in this Public License constitutes or may be interpreted
+ as a limitation upon, or waiver of, any privileges and immunities
+ that apply to the Licensor or You, including from the legal
+ processes of any jurisdiction or authority.
+ .
+ .
+ =======================================================================
+ .
+ Creative Commons is not a party to its public
+ licenses. Notwithstanding, Creative Commons may elect to apply one of
+ its public licenses to material it publishes and in those instances
+ will be considered the “Licensor.” The text of the Creative Commons
+ public licenses is dedicated to the public domain under the CC0 Public
+ Domain Dedication. Except for the limited purpose of indicating that
+ material is shared under a Creative Commons public license or as
+ otherwise permitted by the Creative Commons policies published at
+ creativecommons.org/policies, Creative Commons does not authorize the
+ use of the trademark "Creative Commons" or any other trademark or logo
+ of Creative Commons without its prior written consent including,
+ without limitation, in connection with any unauthorized modifications
+ to any of its public licenses or any other arrangements,
+ understandings, or agreements concerning use of licensed material. For
+ the avoidance of doubt, this paragraph does not form part of the
+ public licenses.
+ .
+ Creative Commons may be contacted at creativecommons.org.
+
+License: FSFAP
+ Copying and distribution of this file, with or without modification, are
+ permitted in any medium without royalty provided the copyright notice
+ and this notice are preserved. This file is offered as-is, without any
+ warranty.
+
+License: FSFULLR
+ This file is free software; the Free Software Foundation gives
+ unlimited permission to copy and/or distribute it, with or without
+ modifications, as long as this notice is preserved.
+
+License: GPL-2+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+ .
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ .
+ On Debian systems, the complete text of the GNU General Public
+ License version 2 can be found in `/usr/share/common-licenses/GPL-2'.
+
+License: LGPL-2.1+
This program is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by the
Free Software Foundation; version 2.1 of the License, or (at
.
The complete text of the GNU General Public License
can be found in /usr/share/common-licenses/LGPL-2.1 file.
- .
usr/include/libloc
usr/lib/*/libloc.so
usr/lib/*/pkgconfig
+usr/share/man/man3
usr/lib/*/libloc.so.*
+usr/share/locale/*/LC_MESSAGES/libloc.mo
loc_as_list_new@LIBLOC_1 0.9.5
loc_as_list_ref@LIBLOC_1 0.9.5
loc_as_list_size@LIBLOC_1 0.9.5
+ loc_as_list_sort@LIBLOC_1 0.9.12
loc_as_list_unref@LIBLOC_1 0.9.5
loc_as_new@LIBLOC_1 0.9.4
loc_as_ref@LIBLOC_1 0.9.4
loc_country_list_new@LIBLOC_1 0.9.5
loc_country_list_ref@LIBLOC_1 0.9.5
loc_country_list_size@LIBLOC_1 0.9.5
+ loc_country_list_sort@LIBLOC_1 0.9.12
loc_country_list_unref@LIBLOC_1 0.9.5
loc_country_new@LIBLOC_1 0.9.4
loc_country_ref@LIBLOC_1 0.9.4
+++ /dev/null
-usr/bin/location-importer
-usr/lib/python3*/site-packages/location/database.py
-usr/lib/python3*/site-packages/location/importer.py
+++ /dev/null
-usr/lib/*/perl/
+++ /dev/null
-usr/lib/python3*/site-packages
-usr/bin/location
+usr/bin
var/lib/location/signing-key.pem
-src/systemd/*.service /lib/systemd/system/
-src/systemd/*.timer /lib/systemd/system/
+lib/systemd/system
+usr/share/man/man8
+++ /dev/null
-man/location.8
--- /dev/null
+#!/bin/sh
+
+set -e
+
+case "$1" in
+ configure)
+ mkdir -p /var/lib/location || true
+ ln -s /usr/share/libloc-location/location.db /var/lib/location/database.db 2>/dev/null || true
+ ;;
+esac
+
+#DEBHELPER#
+
+exit 0
--- /dev/null
+#!/bin/sh
+
+set -e
+
+case "$1" in
+ purge)
+ rm -f /var/lib/location/database.db 2>/dev/null
+ rm -f /var/lib/location/signing-key.pem 2>/dev/null
+ rmdir /var/lib/location || true
+ ;;
+esac
+
+#DEBHELPER#
+
+exit 0
--- /dev/null
+usr/lib/python3*
#!/usr/bin/make -f
-# enable verbose mode
-#export DH_VERBOSE=1
-
-# enable all hardening build flags
export DEB_BUILD_MAINT_OPTIONS=hardening=+all
+export PYBUILD_SYSTEM=custom
+export PYBUILD_CLEAN_ARGS=dh_auto_clean
+export PYBUILD_CONFIGURE_ARGS=intltoolize --force --automake; \
+ PYTHON={interpreter} dh_auto_configure -- \
+ --disable-perl
+export PYBUILD_BUILD_ARGS=dh_auto_build
+export PYBUILD_INSTALL_ARGS=dh_auto_install --destdir={destdir}; \
+ mkdir -p {destdir}/usr/lib/python{version}/dist-packages; \
+ mv {destdir}/usr/lib/python3/dist-packages/_location.so {destdir}/usr/lib/python{version}/dist-packages/_location.so; \
+ rm -f {destdir}/usr/lib/python3/dist-packages/_location.la {destdir}/usr/lib/*/libloc.la
+export PYBUILD_TEST_ARGS=dh_auto_test
%:
- dh $@ --with python3 --with-systemd
-
-override_dh_auto_configure:
- intltoolize --force --automake
- dh_auto_configure -- --disable-perl
-
-override_dh_perl:
- dh_perl -d
-
-override_dh_systemd_enable:
- dh_systemd_enable location-update.timer
-
-override_dh_install:
- dh_install
- # lintian: unknown-file-in-python-module-directory
- rm debian/location-python/usr/lib/python3*/site-packages/_location.la
- # linitan: binaries-have-file-conflict (d/location-importer.install)
- rm debian/location-python/usr/lib/python3*/site-packages/location/database.py
- rm debian/location-python/usr/lib/python3*/site-packages/location/importer.py
+ dh $@ --buildsystem=pybuild
version=4
https://source.ipfire.org/releases/libloc/ \
- @PACKAGE@@ANY_VERSION@@ARCHIVE_EXT@ debian uupdate
+ @PACKAGE@@ANY_VERSION@@ARCHIVE_EXT@
src/libloc.pc.in
-src/python/__init__.py.in
-src/python/database.py
-src/python/downloader.py
-src/python/export.py
-src/python/i18n.py
-src/python/importer.py
-src/python/location-importer.in
-src/python/location.in
-src/python/logger.py
+src/python/location/__init__.py
+src/python/location/database.py
+src/python/location/downloader.py
+src/python/location/export.py
+src/python/location/i18n.py
+src/python/location/importer.py
+src/python/location/logger.py
+src/scripts/location-importer.in
+src/scripts/location.in
src/systemd/location-update.service.in
src/systemd/location-update.timer.in
static PyObject* Database_get_description(DatabaseObject* self) {
const char* description = loc_database_get_description(self->db);
+ if (!description)
+ Py_RETURN_NONE;
return PyUnicode_FromString(description);
}
static PyObject* Database_get_vendor(DatabaseObject* self) {
const char* vendor = loc_database_get_vendor(self->db);
+ if (!vendor)
+ Py_RETURN_NONE;
return PyUnicode_FromString(vendor);
}
static PyObject* Database_get_license(DatabaseObject* self) {
const char* license = loc_database_get_license(self->db);
+ if (!license)
+ Py_RETURN_NONE;
return PyUnicode_FromString(license);
}
# #
###############################################################################
-__version__ = "@VERSION@"
-
# Import everything from the C module
from _location import *
import urllib.parse
import urllib.request
-from . import __version__
-from _location import Database, DATABASE_VERSION_LATEST
+from _location import Database, DATABASE_VERSION_LATEST, __version__
DATABASE_FILENAME = "location.db.xz"
MIRRORS = (
# #
###############################################################################
-import functools
import io
import ipaddress
import logging
else:
self.f = io.StringIO()
+ # Tag
+ self.tag = self._make_tag()
+
# Call any custom initialization
self.init()
def __repr__(self):
return "<%s %s f=%s>" % (self.__class__.__name__, self, self.f)
- @functools.cached_property
- def tag(self):
+ def _make_tag(self):
families = {
socket.AF_INET6 : "6",
socket.AF_INET : "4",
return "%sv%s" % (self.name, families.get(self.family, "?"))
- @functools.cached_property
+ @property
def filename(self):
if self.directory:
return os.path.join(self.directory, "%s.%s" % (self.tag, self.suffix))
def _write_footer(self):
# Jump back to the beginning of the file
- self.f.seek(0)
+ try:
+ self.f.seek(0)
+
+ # If the output stream isn't seekable, we won't try writing the header again
+ except io.UnsupportedOperation:
+ return
# Rewrite the header with better configuration
self._write_header()
import gzip
import logging
+import tempfile
import urllib.request
# Initialise logging
# ],
}
+# List all sources
+SOURCES = set(WHOIS_SOURCES|EXTENDED_SOURCES)
+
class Downloader(object):
def __init__(self):
self.proxy = None
log.info("Using proxy %s" % url)
self.proxy = url
- def request(self, url, data=None, return_blocks=False):
+ def retrieve(self, url, data=None):
+ """
+ This method will fetch the content at the given URL
+ and will return a file-object to a temporary file.
+
+ If the content was compressed, it will be decompressed on the fly.
+ """
+ # Open a temporary file to buffer the downloaded content
+ t = tempfile.SpooledTemporaryFile(max_size=100 * 1024 * 1024)
+
+ # Create a new request
req = urllib.request.Request(url, data=data)
# Configure proxy
if self.proxy:
req.set_proxy(self.proxy, "http")
- return DownloaderContext(self, req, return_blocks=return_blocks)
-
-
-class DownloaderContext(object):
- def __init__(self, downloader, request, return_blocks=False):
- self.downloader = downloader
- self.request = request
-
- # Should we return one block or a single line?
- self.return_blocks = return_blocks
-
- # Save the response object
- self.response = None
-
- def __enter__(self):
- log.info("Retrieving %s..." % self.request.full_url)
+ log.info("Retrieving %s..." % req.full_url)
# Send request
- self.response = urllib.request.urlopen(self.request)
+ res = urllib.request.urlopen(req)
# Log the response headers
log.debug("Response Headers:")
- for header in self.headers:
- log.debug(" %s: %s" % (header, self.get_header(header)))
+ for header in res.headers:
+ log.debug(" %s: %s" % (header, res.headers[header]))
- return self
+ # Write the payload to the temporary file
+ with res as f:
+ while True:
+ buf = f.read(65536)
+ if not buf:
+ break
- def __exit__(self, type, value, traceback):
- pass
+ t.write(buf)
- def __iter__(self):
- """
- Makes the object iterable by going through each block
- """
- if self.return_blocks:
- return iterate_over_blocks(self.body)
+ # Rewind the temporary file
+ t.seek(0)
- return iterate_over_lines(self.body)
+ # Fetch the content type
+ content_type = res.headers.get("Content-Type")
- @property
- def headers(self):
- if self.response:
- return self.response.headers
+ # Decompress any gzipped response on the fly
+ if content_type in ("application/x-gzip", "application/gzip"):
+ t = gzip.GzipFile(fileobj=t, mode="rb")
- def get_header(self, name):
- if self.headers:
- return self.headers.get(name)
+ # Return the temporary file handle
+ return t
- @property
- def body(self):
+ def request_blocks(self, url, data=None):
"""
- Returns a file-like object with the decoded content
- of the response.
+ This method will fetch the data from the URL and return an
+ iterator for each block in the data.
"""
- content_type = self.get_header("Content-Type")
+ # Download the data first
+ t = self.retrieve(url, data=data)
- # Decompress any gzipped response on the fly
- if content_type in ("application/x-gzip", "application/gzip"):
- return gzip.GzipFile(fileobj=self.response, mode="rb")
+ # Then, split it into blocks
+ return iterate_over_blocks(t)
+
+ def request_lines(self, url, data=None):
+ """
+ This method will fetch the data from the URL and return an
+ iterator for each line in the data.
+ """
+ # Download the data first
+ t = self.retrieve(url, data=data)
- # Return the response by default
- return self.response
+ # Then, split it into lines
+ return iterate_over_lines(t)
def read_blocks(f):
block = []
for line in f:
+ # Skip commented lines
+ if line.startswith(b"#") or line.startswith(b"%"):
+ continue
+
# Convert to string
for charset in charsets:
try:
else:
break
- # Skip commented lines
- if line.startswith("#") or line.startswith("%"):
- continue
-
- # Strip line-endings
- line = line.rstrip()
-
# Remove any comments at the end of line
line, hash, comment = line.partition("#")
- if comment:
- # Strip any whitespace before the comment
- line = line.rstrip()
+ # Strip any whitespace at the end of the line
+ line = line.rstrip()
- # If the line is now empty, we move on
- if not line:
- continue
+ # If we cut off some comment and the line is empty, we can skip it
+ if comment and not line:
+ continue
+ # If the line has some content, keep collecting it
if line:
block.append(line)
continue
if (!m)
return NULL;
+ // Version
+ if (PyModule_AddStringConstant(m, "__version__", PACKAGE_VERSION))
+ return NULL;
+
// AS
if (PyType_Ready(&ASType) < 0)
return NULL;
import socket
import sys
import telnetlib
+import urllib.error
# Load our location module
import location
def handle_update_whois(self, ns):
downloader = location.importer.Downloader()
- # Download all sources
- with self.db.transaction():
- # Create some temporary tables to store parsed data
- self.db.execute("""
- CREATE TEMPORARY TABLE _autnums(number integer NOT NULL, organization text NOT NULL, source text NOT NULL)
- ON COMMIT DROP;
- CREATE UNIQUE INDEX _autnums_number ON _autnums(number);
-
- CREATE TEMPORARY TABLE _organizations(handle text NOT NULL, name text NOT NULL, source text NOT NULL)
- ON COMMIT DROP;
- CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
-
- CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL, original_countries text[] NOT NULL, source text NOT NULL)
- ON COMMIT DROP;
- CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
- CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
- """)
+ # Did we run successfully?
+ error = False
- # Remove all previously imported content
- self.db.execute("""
- TRUNCATE TABLE networks;
- """)
+ # Fetch all valid country codes to check parsed networks aganist
+ validcountries = self.countries
- # Fetch all valid country codes to check parsed networks aganist...
- rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
- validcountries = []
+ # Iterate over all potential sources
+ for source in sorted(location.importer.SOURCES):
+ with self.db.transaction():
+ # Create some temporary tables to store parsed data
+ self.db.execute("""
+ CREATE TEMPORARY TABLE _autnums(number integer NOT NULL,
+ organization text NOT NULL, source text NOT NULL) ON COMMIT DROP;
+ CREATE UNIQUE INDEX _autnums_number ON _autnums(number);
- for row in rows:
- validcountries.append(row.country_code)
+ CREATE TEMPORARY TABLE _organizations(handle text NOT NULL,
+ name text NOT NULL, source text NOT NULL) ON COMMIT DROP;
+ CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
- for source_key in location.importer.WHOIS_SOURCES:
- for single_url in location.importer.WHOIS_SOURCES[source_key]:
- with downloader.request(single_url, return_blocks=True) as f:
- for block in f:
- self._parse_block(block, source_key, validcountries)
+ CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL,
+ original_countries text[] NOT NULL, source text NOT NULL)
+ ON COMMIT DROP;
+ CREATE INDEX _rirdata_search ON _rirdata
+ USING BTREE(family(network), masklen(network));
+ CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
+ """)
- # Process all parsed networks from every RIR we happen to have access to,
- # insert the largest network chunks into the networks table immediately...
- families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
+ # Remove all previously imported content
+ self.db.execute("DELETE FROM networks WHERE source = %s", source)
- for family in (row.family for row in families):
- smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
+ try:
+ # Fetch WHOIS sources
+ for url in location.importer.WHOIS_SOURCES.get(source, []):
+ for block in downloader.request_blocks(url):
+ self._parse_block(block, source, validcountries)
+
+ # Fetch extended sources
+ for url in location.importer.EXTENDED_SOURCES.get(source, []):
+ for line in downloader.request_lines(url):
+ self._parse_line(line, source, validcountries)
+ except urllib.error.URLError as e:
+ log.error("Could not retrieve data from %s: %s" % (source, e))
+ error = True
+
+ # Continue with the next source
+ continue
- self.db.execute("INSERT INTO networks(network, country, original_countries, source) \
- SELECT network, country, original_countries, source FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
+ # Process all parsed networks from every RIR we happen to have access to,
+ # insert the largest network chunks into the networks table immediately...
+ families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata \
+ ORDER BY family(network)")
- # ... determine any other prefixes for this network family, ...
- prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
- WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
+ for family in (row.family for row in families):
+ # Fetch the smallest mask length in our data set
+ smallest = self.db.get("""
+ SELECT
+ MIN(
+ masklen(network)
+ ) AS prefix
+ FROM
+ _rirdata
+ WHERE
+ family(network) = %s""",
+ family,
+ )
- # ... and insert networks with this prefix in case they provide additional
- # information (i. e. subnet of a larger chunk with a different country)
- for prefix in (row.prefix for row in prefixes):
+ # Copy all networks
self.db.execute("""
- WITH candidates AS (
- SELECT
- _rirdata.network,
- _rirdata.country,
- _rirdata.original_countries,
- _rirdata.source
- FROM
- _rirdata
- WHERE
- family(_rirdata.network) = %s
- AND
- masklen(_rirdata.network) = %s
- ),
- filtered AS (
- SELECT
- DISTINCT ON (c.network)
- c.network,
- c.country,
- c.original_countries,
- c.source,
- masklen(networks.network),
- networks.country AS parent_country
- FROM
- candidates c
- LEFT JOIN
- networks
- ON
- c.network << networks.network
- ORDER BY
- c.network,
- masklen(networks.network) DESC NULLS LAST
- )
INSERT INTO
- networks(network, country, original_countries, source)
+ networks
+ (
+ network,
+ country,
+ original_countries,
+ source
+ )
SELECT
network,
country,
original_countries,
source
FROM
- filtered
+ _rirdata
WHERE
- parent_country IS NULL
- OR
- country <> parent_country
- ON CONFLICT DO NOTHING""",
- family, prefix,
+ masklen(network) = %s
+ AND
+ family(network) = %s
+ ON CONFLICT DO
+ NOTHING""",
+ smallest.prefix,
+ family,
)
- self.db.execute("""
- INSERT INTO autnums(number, name, source)
- SELECT _autnums.number, _organizations.name, _organizations.source FROM _autnums
- JOIN _organizations ON _autnums.organization = _organizations.handle
- ON CONFLICT (number) DO UPDATE SET name = excluded.name;
- """)
+ # ... determine any other prefixes for this network family, ...
+ prefixes = self.db.query("""
+ SELECT
+ DISTINCT masklen(network) AS prefix
+ FROM
+ _rirdata
+ WHERE
+ family(network) = %s
+ ORDER BY
+ masklen(network) ASC
+ OFFSET 1""",
+ family,
+ )
- # Download all extended sources
- for source_key in location.importer.EXTENDED_SOURCES:
- for single_url in location.importer.EXTENDED_SOURCES[source_key]:
- with self.db.transaction():
- # Download data
- with downloader.request(single_url) as f:
- for line in f:
- self._parse_line(line, source_key, validcountries)
+ # ... and insert networks with this prefix in case they provide additional
+ # information (i. e. subnet of a larger chunk with a different country)
+ for prefix in (row.prefix for row in prefixes):
+ self.db.execute("""
+ WITH candidates AS (
+ SELECT
+ _rirdata.network,
+ _rirdata.country,
+ _rirdata.original_countries,
+ _rirdata.source
+ FROM
+ _rirdata
+ WHERE
+ family(_rirdata.network) = %s
+ AND
+ masklen(_rirdata.network) = %s
+ ),
+ filtered AS (
+ SELECT
+ DISTINCT ON (c.network)
+ c.network,
+ c.country,
+ c.original_countries,
+ c.source,
+ masklen(networks.network),
+ networks.country AS parent_country
+ FROM
+ candidates c
+ LEFT JOIN
+ networks
+ ON
+ c.network << networks.network
+ ORDER BY
+ c.network,
+ masklen(networks.network) DESC NULLS LAST
+ )
+ INSERT INTO
+ networks(network, country, original_countries, source)
+ SELECT
+ network,
+ country,
+ original_countries,
+ source
+ FROM
+ filtered
+ WHERE
+ parent_country IS NULL
+ OR
+ country <> parent_country
+ ON CONFLICT DO NOTHING""",
+ family, prefix,
+ )
+
+ self.db.execute("""
+ INSERT INTO autnums(number, name, source)
+ SELECT _autnums.number, _organizations.name, _organizations.source FROM _autnums
+ JOIN _organizations ON _autnums.organization = _organizations.handle
+ ON CONFLICT (number) DO UPDATE SET name = excluded.name;
+ """)
# Download and import (technical) AS names from ARIN
- self._import_as_names_from_arin()
+ with self.db.transaction():
+ self._import_as_names_from_arin()
+
+ # Return a non-zero exit code for errors
+ return 1 if error else 0
def _check_parsed_network(self, network):
"""
# technical, not intended for human consumption, as description fields in
# organisation handles for other RIRs are - however, this is what we have got,
# and in some cases, it might be still better than nothing)
- with downloader.request("https://ftp.arin.net/info/asn.txt", return_blocks=False) as f:
- for line in f:
- # Convert binary line to string...
- line = str(line)
-
- # ... valid lines start with a space, followed by the number of the Autonomous System ...
- if not line.startswith(" "):
- continue
+ for line in downloader.request_lines("https://ftp.arin.net/info/asn.txt"):
+ # Valid lines start with a space, followed by the number of the Autonomous System ...
+ if not line.startswith(" "):
+ continue
- # Split line and check if there is a valid ASN in it...
- asn, name = line.split()[0:2]
+ # Split line and check if there is a valid ASN in it...
+ asn, name = line.split()[0:2]
- try:
- asn = int(asn)
- except ValueError:
- log.debug("Skipping ARIN AS names line not containing an integer for ASN")
- continue
+ try:
+ asn = int(asn)
+ except ValueError:
+ log.debug("Skipping ARIN AS names line not containing an integer for ASN")
+ continue
- # Filter invalid ASNs...
- if not self._check_parsed_asn(asn):
- continue
+ # Filter invalid ASNs...
+ if not self._check_parsed_asn(asn):
+ continue
- # Skip any AS name that appears to be a placeholder for a different RIR or entity...
- if re.match(r"^(ASN-BLK|)(AFCONC|AFRINIC|APNIC|ASNBLK|DNIC|LACNIC|RIPE|IANA)(?:\d?$|\-)", name):
- continue
+ # Skip any AS name that appears to be a placeholder for a different RIR or entity...
+ if re.match(r"^(ASN-BLK|)(AFCONC|AFRINIC|APNIC|ASNBLK|LACNIC|RIPE|IANA)(?:\d?$|\-)", name):
+ continue
- # Bail out in case the AS name contains anything we do not expect here...
- if re.search(r"[^a-zA-Z0-9-_]", name):
- log.debug("Skipping ARIN AS name for %s containing invalid characters: %s" % \
- (asn, name))
+ # Bail out in case the AS name contains anything we do not expect here...
+ if re.search(r"[^a-zA-Z0-9-_]", name):
+ log.debug("Skipping ARIN AS name for %s containing invalid characters: %s" % \
+ (asn, name))
- # Things look good here, run INSERT statement and skip this one if we already have
- # a (better?) name for this Autonomous System...
- self.db.execute("""
- INSERT INTO autnums(
- number,
- name,
- source
- ) VALUES (%s, %s, %s)
- ON CONFLICT (number) DO NOTHING""",
- asn,
+ # Things look good here, run INSERT statement and skip this one if we already have
+ # a (better?) name for this Autonomous System...
+ self.db.execute("""
+ INSERT INTO autnums(
+ number,
name,
- "ARIN",
- )
+ source
+ ) VALUES (%s, %s, %s)
+ ON CONFLICT (number) DO NOTHING""",
+ asn,
+ name,
+ "ARIN",
+ )
def handle_update_announcements(self, ns):
server = ns.server[0]
def handle_update_overrides(self, ns):
with self.db.transaction():
- # Drop all data that we have
+ # Only drop manually created overrides, as we can be reasonably sure to have them,
+ # and preserve the rest. If appropriate, it is deleted by correspondent functions.
self.db.execute("""
- TRUNCATE TABLE autnum_overrides;
- TRUNCATE TABLE network_overrides;
+ DELETE FROM autnum_overrides WHERE source = 'manual';
+ DELETE FROM network_overrides WHERE source = 'manual';
""")
# Update overrides for various cloud providers big enough to publish their own IP
downloader = location.importer.Downloader()
try:
- with downloader.request("https://ip-ranges.amazonaws.com/ip-ranges.json", return_blocks=False) as f:
- aws_ip_dump = json.load(f.body)
+ # Fetch IP ranges
+ f = downloader.retrieve("https://ip-ranges.amazonaws.com/ip-ranges.json")
+
+ # Parse downloaded file
+ aws_ip_dump = json.load(f)
except Exception as e:
log.error("unable to preprocess Amazon AWS IP ranges: %s" % e)
return
+ # At this point, we can assume the downloaded file to be valid
+ self.db.execute("""
+ DELETE FROM network_overrides WHERE source = 'Amazon AWS IP feed';
+ """)
+
# XXX: Set up a dictionary for mapping a region name to a country. Unfortunately,
# there seems to be no machine-readable version available of this other than
# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html
]
for url in ip_urls:
- try:
- with downloader.request(url, return_blocks=False) as f:
- fcontent = f.body.readlines()
- except Exception as e:
- log.error("Unable to download Spamhaus DROP URL %s: %s" % (url, e))
- return
+ # Fetch IP list
+ f = downloader.retrieve(url)
+
+ # Split into lines
+ fcontent = f.readlines()
+
+ # Conduct a very basic sanity check to rule out CDN issues causing bogus DROP
+ # downloads.
+ if len(fcontent) > 10:
+ self.db.execute("""
+ DELETE FROM autnum_overrides WHERE source = 'Spamhaus ASN-DROP list';
+ DELETE FROM network_overrides WHERE source = 'Spamhaus DROP lists';
+ """)
+ else:
+ log.error("Spamhaus DROP URL %s returned likely bogus file, ignored" % url)
+ continue
# Iterate through every line, filter comments and add remaining networks to
# the override table in case they are valid...
with self.db.transaction():
for sline in fcontent:
-
# The response is assumed to be encoded in UTF-8...
sline = sline.decode("utf-8")
)
for url in asn_urls:
- try:
- with downloader.request(url, return_blocks=False) as f:
- fcontent = f.body.readlines()
- except Exception as e:
- log.error("Unable to download Spamhaus DROP URL %s: %s" % (url, e))
- return
+ # Fetch URL
+ f = downloader.retrieve(url)
# Iterate through every line, filter comments and add remaining ASNs to
# the override table in case they are valid...
with self.db.transaction():
- for sline in fcontent:
-
+ for sline in t.readlines():
# The response is assumed to be encoded in UTF-8...
sline = sline.decode("utf-8")
# Default to None
return None
+ @property
+ def countries(self):
+ # Fetch all valid country codes to check parsed networks aganist
+ rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
+
+ # Return all countries
+ return [row.country_code for row in rows]
+
def handle_import_countries(self, ns):
with self.db.transaction():
# Drop all data that we have
return 1
# Success
- log.debug("Database successfully verified")
+ log.info("Database successfully verified")
return 0
def __get_output_formatter(self, ns):
projects / location / debian / libloc.git / commitdiff
